From 3d028308ef1e7ef51db451b40331efcf51daa053 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Thu, 28 Jul 2022 14:01:50 -0700 Subject: [PATCH] Cleanup CRI files Signed-off-by: Maksym Pavlenko --- pkg/cri/sbserver/helpers_linux.go | 13 - pkg/cri/sbserver/helpers_linux_test.go | 100 ---- .../sbserver/helpers_selinux_linux_test.go | 157 ------ pkg/cri/sbserver/podsandbox/helpers.go | 125 ----- pkg/cri/sbserver/podsandbox/helpers_linux.go | 22 - pkg/cri/sbserver/podsandbox/helpers_other.go | 5 - .../sbserver/podsandbox/helpers_windows.go | 135 ----- pkg/cri/sbserver/podsandbox/sandbox_run.go | 4 +- pkg/cri/sbserver/sandbox_run_linux.go | 291 ---------- pkg/cri/sbserver/sandbox_run_linux_test.go | 526 ------------------ pkg/cri/sbserver/sandbox_run_other.go | 20 - pkg/cri/sbserver/sandbox_run_other_test.go | 36 -- pkg/cri/sbserver/sandbox_run_test.go | 348 ------------ pkg/cri/sbserver/sandbox_run_windows.go | 81 --- pkg/cri/sbserver/sandbox_run_windows_test.go | 108 ---- 15 files changed, 2 insertions(+), 1969 deletions(-) delete mode 100644 pkg/cri/sbserver/helpers_linux_test.go delete mode 100644 pkg/cri/sbserver/helpers_selinux_linux_test.go delete mode 100644 pkg/cri/sbserver/sandbox_run_linux_test.go delete mode 100644 pkg/cri/sbserver/sandbox_run_other_test.go delete mode 100644 pkg/cri/sbserver/sandbox_run_windows_test.go diff --git a/pkg/cri/sbserver/helpers_linux.go b/pkg/cri/sbserver/helpers_linux.go index 060e7631e..655b96f0e 100644 --- a/pkg/cri/sbserver/helpers_linux.go +++ b/pkg/cri/sbserver/helpers_linux.go @@ -35,16 +35,11 @@ import ( "github.com/containerd/containerd/pkg/seutil" "github.com/moby/sys/mountinfo" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/opencontainers/selinux/go-selinux/label" "golang.org/x/sys/unix" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) const ( - // defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938). - defaultSandboxOOMAdj = -998 - // defaultShmSize is the default size of the sandbox shm. - defaultShmSize = int64(1024 * 1024 * 64) // relativeRootfsPath is the rootfs path relative to bundle path. relativeRootfsPath = "rootfs" // devShm is the default path of /dev/shm. @@ -115,14 +110,6 @@ func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) { return labels, nil } -func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) { - labels, err := toLabel(selinuxOpts) - if err != nil { - return "", "", err - } - return label.InitLabels(labels) -} - func checkSelinuxLevel(level string) error { if len(level) == 0 { return nil diff --git a/pkg/cri/sbserver/helpers_linux_test.go b/pkg/cri/sbserver/helpers_linux_test.go deleted file mode 100644 index b0ded7c81..000000000 --- a/pkg/cri/sbserver/helpers_linux_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "golang.org/x/sys/unix" -) - -func TestGetCgroupsPath(t *testing.T) { - testID := "test-id" - for desc, test := range map[string]struct { - cgroupsParent string - expected string - }{ - "should support regular cgroup path": { - cgroupsParent: "/a/b", - expected: "/a/b/test-id", - }, - "should support systemd cgroup path": { - cgroupsParent: "/a.slice/b.slice", - expected: "b.slice:cri-containerd:test-id", - }, - "should support tailing slash for regular cgroup path": { - cgroupsParent: "/a/b/", - expected: "/a/b/test-id", - }, - "should support tailing slash for systemd cgroup path": { - cgroupsParent: "/a.slice/b.slice/", - expected: "b.slice:cri-containerd:test-id", - }, - "should treat root cgroup as regular cgroup path": { - cgroupsParent: "/", - expected: "/test-id", - }, - } { - t.Run(desc, func(t *testing.T) { - got := getCgroupsPath(test.cgroupsParent, testID) - assert.Equal(t, test.expected, got) - }) - } -} - -func TestEnsureRemoveAllWithMount(t *testing.T) { - if os.Getuid() != 0 { - t.Skip("skipping test that requires root") - } - - var err error - dir1 := t.TempDir() - dir2 := t.TempDir() - - bindDir := filepath.Join(dir1, "bind") - if err := os.MkdirAll(bindDir, 0755); err != nil { - t.Fatal(err) - } - - if err := unix.Mount(dir2, bindDir, "none", unix.MS_BIND, ""); err != nil { - t.Fatal(err) - } - - done := make(chan struct{}) - go func() { - err = ensureRemoveAll(context.Background(), dir1) - close(done) - }() - - select { - case <-done: - if err != nil { - t.Fatal(err) - } - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for EnsureRemoveAll to finish") - } - - if _, err := os.Stat(dir1); !os.IsNotExist(err) { - t.Fatalf("expected %q to not exist", dir1) - } -} diff --git a/pkg/cri/sbserver/helpers_selinux_linux_test.go b/pkg/cri/sbserver/helpers_selinux_linux_test.go deleted file mode 100644 index d925235b1..000000000 --- a/pkg/cri/sbserver/helpers_selinux_linux_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "testing" - - "github.com/opencontainers/selinux/go-selinux" - "github.com/stretchr/testify/assert" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" -) - -func TestInitSelinuxOpts(t *testing.T) { - if !selinux.GetEnabled() { - t.Skip("selinux is not enabled") - } - - for desc, test := range map[string]struct { - selinuxOpt *runtime.SELinuxOption - processLabel string - mountLabel string - expectErr bool - }{ - "Should return empty strings for processLabel and mountLabel when selinuxOpt is nil": { - selinuxOpt: nil, - processLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}", - mountLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}", - }, - "Should overlay fields on processLabel when selinuxOpt has been initialized partially": { - selinuxOpt: &runtime.SELinuxOption{ - User: "", - Role: "user_r", - Type: "", - Level: "s0:c1,c2", - }, - processLabel: "system_u:user_r:(container_file_t|svirt_lxc_net_t):s0:c1,c2", - mountLabel: "system_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2", - }, - "Should be resolved correctly when selinuxOpt has been initialized completely": { - selinuxOpt: &runtime.SELinuxOption{ - User: "user_u", - Role: "user_r", - Type: "user_t", - Level: "s0:c1,c2", - }, - processLabel: "user_u:user_r:user_t:s0:c1,c2", - mountLabel: "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2", - }, - "Should be resolved correctly when selinuxOpt has been initialized with level=''": { - selinuxOpt: &runtime.SELinuxOption{ - User: "user_u", - Role: "user_r", - Type: "user_t", - Level: "", - }, - processLabel: "user_u:user_r:user_t:s0:c[0-9]{1,3},c[0-9]{1,3}", - mountLabel: "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0", - }, - "Should return error when the format of 'level' is not correct": { - selinuxOpt: &runtime.SELinuxOption{ - User: "user_u", - Role: "user_r", - Type: "user_t", - Level: "s0,c1,c2", - }, - expectErr: true, - }, - } { - t.Run(desc, func(t *testing.T) { - processLabel, mountLabel, err := initLabelsFromOpt(test.selinuxOpt) - if test.expectErr { - assert.Error(t, err) - } else { - assert.Regexp(t, test.processLabel, processLabel) - assert.Regexp(t, test.mountLabel, mountLabel) - } - }) - } -} - -func TestCheckSelinuxLevel(t *testing.T) { - for desc, test := range map[string]struct { - level string - expectNoMatch bool - }{ - "s0": { - level: "s0", - }, - "s0-s0": { - level: "s0-s0", - }, - "s0:c0": { - level: "s0:c0", - }, - "s0:c0.c3": { - level: "s0:c0.c3", - }, - "s0:c0,c3": { - level: "s0:c0,c3", - }, - "s0-s0:c0,c3": { - level: "s0-s0:c0,c3", - }, - "s0-s0:c0,c3.c6": { - level: "s0-s0:c0,c3.c6", - }, - "s0-s0:c0,c3.c6,c8.c10": { - level: "s0-s0:c0,c3.c6,c8.c10", - }, - "s0-s0:c0,c3.c6,c8,c10": { - level: "s0-s0:c0,c3.c6", - }, - "s0,c0,c3": { - level: "s0,c0,c3", - expectNoMatch: true, - }, - "s0:c0.c3.c6": { - level: "s0:c0.c3.c6", - expectNoMatch: true, - }, - "s0-s0,c0,c3": { - level: "s0-s0,c0,c3", - expectNoMatch: true, - }, - "s0-s0:c0.c3.c6": { - level: "s0-s0:c0.c3.c6", - expectNoMatch: true, - }, - "s0-s0:c0,c3.c6.c8": { - level: "s0-s0:c0,c3.c6.c8", - expectNoMatch: true, - }, - } { - t.Run(desc, func(t *testing.T) { - err := checkSelinuxLevel(test.level) - if test.expectNoMatch { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - }) - } -} diff --git a/pkg/cri/sbserver/podsandbox/helpers.go b/pkg/cri/sbserver/podsandbox/helpers.go index 1705fe2d2..0844e72e6 100644 --- a/pkg/cri/sbserver/podsandbox/helpers.go +++ b/pkg/cri/sbserver/podsandbox/helpers.go @@ -30,9 +30,7 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/oci" criconfig "github.com/containerd/containerd/pkg/cri/config" - containerstore "github.com/containerd/containerd/pkg/cri/store/container" imagestore "github.com/containerd/containerd/pkg/cri/store/image" - sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/pkg/cri/util" runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1" "github.com/containerd/containerd/plugin" @@ -46,80 +44,26 @@ import ( runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" imagedigest "github.com/opencontainers/go-digest" "github.com/pelletier/go-toml" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) const ( - // errorStartReason is the exit reason when fails to start container. - errorStartReason = "StartError" - // errorStartExitCode is the exit code when fails to start container. - // 128 is the same with Docker's behavior. - // TODO(windows): Figure out what should be used for windows. - errorStartExitCode = 128 - // completeExitReason is the exit reason when container exits with code 0. - completeExitReason = "Completed" - // errorExitReason is the exit reason when container exits with code non-zero. - errorExitReason = "Error" - // oomExitReason is the exit reason when process in container is oom killed. - oomExitReason = "OOMKilled" // sandboxesDir contains all sandbox root. A sandbox root is the running // directory of the sandbox, all files created for the sandbox will be // placed under this directory. sandboxesDir = "sandboxes" - // containersDir contains all container root. - containersDir = "containers" - // Delimiter used to construct container/sandbox names. - nameDelimiter = "_" - // criContainerdPrefix is common prefix for cri-containerd criContainerdPrefix = "io.cri-containerd" // containerKindLabel is a label key indicating container is sandbox container or application container containerKindLabel = criContainerdPrefix + ".kind" // containerKindSandbox is a label value indicating container is sandbox container containerKindSandbox = "sandbox" - // containerKindContainer is a label value indicating container is application container - containerKindContainer = "container" - // imageLabelKey is the label key indicating the image is managed by cri plugin. - imageLabelKey = criContainerdPrefix + ".image" - // imageLabelValue is the label value indicating the image is managed by cri plugin. - imageLabelValue = "managed" // sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata" - // containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest - containerMetadataExtension = criContainerdPrefix + ".container.metadata" - - // defaultIfName is the default network interface for the pods - defaultIfName = "eth0" - // runtimeRunhcsV1 is the runtime type for runhcs. runtimeRunhcsV1 = "io.containerd.runhcs.v1" ) -// makeSandboxName generates sandbox name from sandbox metadata. The name -// generated is unique as long as sandbox metadata is unique. -func makeSandboxName(s *runtime.PodSandboxMetadata) string { - return strings.Join([]string{ - s.Name, // 0 - s.Namespace, // 1 - s.Uid, // 2 - fmt.Sprintf("%d", s.Attempt), // 3 - }, nameDelimiter) -} - -// makeContainerName generates container name from sandbox and container metadata. -// The name generated is unique as long as the sandbox container combination is -// unique. -func makeContainerName(c *runtime.ContainerMetadata, s *runtime.PodSandboxMetadata) string { - return strings.Join([]string{ - c.Name, // 0: container name - s.Name, // 1: pod name - s.Namespace, // 2: pod namespace - s.Uid, // 3: pod uid - fmt.Sprintf("%d", c.Attempt), // 4: attempt number of creating the container - }, nameDelimiter) -} - // getSandboxRootDir returns the root directory for managing sandbox files, // e.g. hosts files. func (c *Controller) getSandboxRootDir(id string) string { @@ -132,23 +76,6 @@ func (c *Controller) getVolatileSandboxRootDir(id string) string { return filepath.Join(c.config.StateDir, sandboxesDir, id) } -// getContainerRootDir returns the root directory for managing container files, -// e.g. state checkpoint. -func (c *Controller) getContainerRootDir(id string) string { - return filepath.Join(c.config.RootDir, containersDir, id) -} - -// getVolatileContainerRootDir returns the root directory for managing volatile container files, -// e.g. named pipes. -func (c *Controller) getVolatileContainerRootDir(id string) string { - return filepath.Join(c.config.StateDir, containersDir, id) -} - -// criContainerStateToString formats CRI container state to string. -func criContainerStateToString(state runtime.ContainerState) string { - return runtime.ContainerState_name[int32(state)] -} - // getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference. func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) { var repoTag, repoDigest string @@ -192,22 +119,6 @@ func getUserFromImage(user string) (*int64, string) { return &uid, "" } -// isInCRIMounts checks whether a destination is in CRI mount list. -func isInCRIMounts(dst string, mounts []*runtime.Mount) bool { - for _, m := range mounts { - if filepath.Clean(m.ContainerPath) == filepath.Clean(dst) { - return true - } - } - return false -} - -// filterLabel returns a label filter. Use `%q` here because containerd -// filter needs extra quote to work properly. -func filterLabel(k, v string) string { - return fmt.Sprintf("labels.%q==%q", k, v) -} - // buildLabel builds the labels from config to be passed to containerd func buildLabels(configLabels, imageConfigLabels map[string]string, containerType string) map[string]string { labels := make(map[string]string) @@ -229,16 +140,6 @@ func buildLabels(configLabels, imageConfigLabels map[string]string, containerTyp return labels } -// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config. -func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig { - return &runtime.AuthConfig{ - Username: a.Username, - Password: a.Password, - Auth: a.Auth, - IdentityToken: a.IdentityToken, - } -} - // parseImageReferences parses a list of arbitrary image references and returns // the repotags and repodigests func parseImageReferences(refs []string) ([]string, []string) { @@ -310,32 +211,6 @@ func getRuntimeOptions(c containers.Container) (interface{}, error) { return opts, nil } -const ( - // unknownExitCode is the exit code when exit reason is unknown. - unknownExitCode = 255 - // unknownExitReason is the exit reason when exit reason is unknown. - unknownExitReason = "Unknown" -) - -// unknownContainerStatus returns the default container status when its status is unknown. -func unknownContainerStatus() containerstore.Status { - return containerstore.Status{ - CreatedAt: 0, - StartedAt: 0, - FinishedAt: 0, - ExitCode: unknownExitCode, - Reason: unknownExitReason, - Unknown: true, - } -} - -// unknownSandboxStatus returns the default sandbox status when its status is unknown. -func unknownSandboxStatus() sandboxstore.Status { - return sandboxstore.Status{ - State: sandboxstore.StateUnknown, - } -} - // getPassthroughAnnotations filters requested pod annotations by comparing // against permitted annotations for the given runtime. func getPassthroughAnnotations(podAnnotations map[string]string, diff --git a/pkg/cri/sbserver/podsandbox/helpers_linux.go b/pkg/cri/sbserver/podsandbox/helpers_linux.go index 861514787..a0a33b2bc 100644 --- a/pkg/cri/sbserver/podsandbox/helpers_linux.go +++ b/pkg/cri/sbserver/podsandbox/helpers_linux.go @@ -30,7 +30,6 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/pkg/apparmor" "github.com/containerd/containerd/pkg/seccomp" "github.com/containerd/containerd/pkg/seutil" "github.com/moby/sys/mountinfo" @@ -51,12 +50,8 @@ const ( devShm = "/dev/shm" // etcHosts is the default path of /etc/hosts file. etcHosts = "/etc/hosts" - // etcHostname is the default path of /etc/hostname file. - etcHostname = "/etc/hostname" // resolvConfPath is the abs path of resolv.conf on host or container. resolvConfPath = "/etc/resolv.conf" - // hostnameEnv is the key for HOSTNAME env. - hostnameEnv = "HOSTNAME" ) // getCgroupsPath generates container cgroups path. @@ -138,27 +133,10 @@ func checkSelinuxLevel(level string) error { return nil } -// apparmorEnabled returns true if apparmor is enabled, supported by the host, -// if apparmor_parser is installed, and if we are not running docker-in-docker. -func (c *Controller) apparmorEnabled() bool { - if c.config.DisableApparmor { - return false - } - return apparmor.HostSupports() -} - func (c *Controller) seccompEnabled() bool { return seccomp.IsEnabled() } -// openLogFile opens/creates a container log file. -func openLogFile(path string) (*os.File, error) { - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return nil, err - } - return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) -} - // unmountRecursive unmounts the target and all mounts underneath, starting with // the deepest mount first. func unmountRecursive(ctx context.Context, target string) error { diff --git a/pkg/cri/sbserver/podsandbox/helpers_other.go b/pkg/cri/sbserver/podsandbox/helpers_other.go index b66c26a95..7ed1cb9ad 100644 --- a/pkg/cri/sbserver/podsandbox/helpers_other.go +++ b/pkg/cri/sbserver/podsandbox/helpers_other.go @@ -26,11 +26,6 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" ) -// openLogFile opens/creates a container log file. -func openLogFile(path string) (*os.File, error) { - return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) -} - // ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can // often be remedied. // Only use `ensureRemoveAll` if you really want to make every effort to remove diff --git a/pkg/cri/sbserver/podsandbox/helpers_windows.go b/pkg/cri/sbserver/podsandbox/helpers_windows.go index a98563fd2..044ea63a9 100644 --- a/pkg/cri/sbserver/podsandbox/helpers_windows.go +++ b/pkg/cri/sbserver/podsandbox/helpers_windows.go @@ -19,145 +19,10 @@ package podsandbox import ( "context" "os" - "path/filepath" - "syscall" "github.com/opencontainers/runtime-spec/specs-go" ) -// openLogFile opens/creates a container log file. -// It specifies `FILE_SHARE_DELETE` option to make sure -// log files can be rotated by kubelet. -// TODO(windows): Use golang support after 1.14. (https://github.com/golang/go/issues/32088) -func openLogFile(path string) (*os.File, error) { - path = fixLongPath(path) - if len(path) == 0 { - return nil, syscall.ERROR_FILE_NOT_FOUND - } - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return nil, err - } - createmode := uint32(syscall.OPEN_ALWAYS) - access := uint32(syscall.FILE_APPEND_DATA) - sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) - h, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) - if err != nil { - return nil, err - } - return os.NewFile(uintptr(h), path), nil -} - -// Copyright (c) 2009 The Go Authors. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// fixLongPath returns the extended-length (\\?\-prefixed) form of -// path when needed, in order to avoid the default 260 character file -// path limit imposed by Windows. If path is not easily converted to -// the extended-length form (for example, if path is a relative path -// or contains .. elements), or is short enough, fixLongPath returns -// path unmodified. -// -// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath -// -// This is copied from https://golang.org/src/path/filepath/path_windows.go. -func fixLongPath(path string) string { - // Do nothing (and don't allocate) if the path is "short". - // Empirically (at least on the Windows Server 2013 builder), - // the kernel is arbitrarily okay with < 248 bytes. That - // matches what the docs above say: - // "When using an API to create a directory, the specified - // path cannot be so long that you cannot append an 8.3 file - // name (that is, the directory name cannot exceed MAX_PATH - // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. - // - // The MSDN docs appear to say that a normal path that is 248 bytes long - // will work; empirically the path must be less then 248 bytes long. - if len(path) < 248 { - // Don't fix. (This is how Go 1.7 and earlier worked, - // not automatically generating the \\?\ form) - return path - } - - // The extended form begins with \\?\, as in - // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. - // The extended form disables evaluation of . and .. path - // elements and disables the interpretation of / as equivalent - // to \. The conversion here rewrites / to \ and elides - // . elements as well as trailing or duplicate separators. For - // simplicity it avoids the conversion entirely for relative - // paths or paths containing .. elements. For now, - // \\server\share paths are not converted to - // \\?\UNC\server\share paths because the rules for doing so - // are less well-specified. - if len(path) >= 2 && path[:2] == `\\` { - // Don't canonicalize UNC paths. - return path - } - if !filepath.IsAbs(path) { - // Relative path - return path - } - - const prefix = `\\?` - - pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) - copy(pathbuf, prefix) - n := len(path) - r, w := 0, len(prefix) - for r < n { - switch { - case os.IsPathSeparator(path[r]): - // empty block - r++ - case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): - // /./ - r++ - case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): - // /../ is currently unhandled - return path - default: - pathbuf[w] = '\\' - w++ - for ; r < n && !os.IsPathSeparator(path[r]); r++ { - pathbuf[w] = path[r] - w++ - } - } - } - // A drive's root directory needs a trailing \ - if w == len(`\\?\c:`) { - pathbuf[w] = '\\' - w++ - } - return string(pathbuf[:w]) -} - // ensureRemoveAll is a wrapper for os.RemoveAll on Windows. func ensureRemoveAll(_ context.Context, dir string) error { return os.RemoveAll(dir) diff --git a/pkg/cri/sbserver/podsandbox/sandbox_run.go b/pkg/cri/sbserver/podsandbox/sandbox_run.go index 542ccef92..0a2111554 100644 --- a/pkg/cri/sbserver/podsandbox/sandbox_run.go +++ b/pkg/cri/sbserver/podsandbox/sandbox_run.go @@ -84,10 +84,10 @@ func (c *Controller) Start(ctx context.Context, id string) (_ uint32, retErr err } log.G(ctx).WithField("podsandboxid", id).Debugf("sandbox container spec: %#+v", spew.NewFormatter(spec)) - processLabel := spec.Process.SelinuxLabel + metadata.ProcessLabel = spec.Process.SelinuxLabel defer func() { if retErr != nil { - selinux.ReleaseLabel(processLabel) + selinux.ReleaseLabel(metadata.ProcessLabel) } }() diff --git a/pkg/cri/sbserver/sandbox_run_linux.go b/pkg/cri/sbserver/sandbox_run_linux.go index 461559d39..2b41c2ec0 100644 --- a/pkg/cri/sbserver/sandbox_run_linux.go +++ b/pkg/cri/sbserver/sandbox_run_linux.go @@ -19,305 +19,14 @@ package sbserver import ( "fmt" "os" - "strconv" - "strings" "github.com/containerd/containerd" - "github.com/containerd/containerd/oci" "github.com/containerd/containerd/plugin" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/opencontainers/selinux/go-selinux" - "golang.org/x/sys/unix" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - "github.com/containerd/containerd/pkg/cri/annotations" - customopts "github.com/containerd/containerd/pkg/cri/opts" osinterface "github.com/containerd/containerd/pkg/os" - "github.com/containerd/containerd/pkg/userns" ) -func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig, - imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) { - // Creates a spec Generator with the default spec. - // TODO(random-liu): [P1] Compare the default settings with docker and containerd default. - specOpts := []oci.SpecOpts{ - oci.WithoutRunMount, - customopts.WithoutDefaultSecuritySettings, - customopts.WithRelativeRoot(relativeRootfsPath), - oci.WithEnv(imageConfig.Env), - oci.WithRootFSReadonly(), - oci.WithHostname(config.GetHostname()), - } - if imageConfig.WorkingDir != "" { - specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir)) - } - - if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 { - // Pause image must have entrypoint or cmd. - return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig) - } - specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...)) - - // Set cgroups parent. - if c.config.DisableCgroup { - specOpts = append(specOpts, customopts.WithDisabledCgroups) - } else { - if config.GetLinux().GetCgroupParent() != "" { - cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id) - specOpts = append(specOpts, oci.WithCgroup(cgroupsPath)) - } - } - - // When cgroup parent is not set, containerd-shim will create container in a child cgroup - // of the cgroup itself is in. - // TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified. - - // Set namespace options. - var ( - securityContext = config.GetLinux().GetSecurityContext() - nsOptions = securityContext.GetNamespaceOptions() - ) - if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE { - specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace)) - specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.UTSNamespace)) - } else { - specOpts = append(specOpts, oci.WithLinuxNamespace( - runtimespec.LinuxNamespace{ - Type: runtimespec.NetworkNamespace, - Path: nsPath, - })) - } - if nsOptions.GetPid() == runtime.NamespaceMode_NODE { - specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace)) - } - if nsOptions.GetIpc() == runtime.NamespaceMode_NODE { - specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace)) - } - - // It's fine to generate the spec before the sandbox /dev/shm - // is actually created. - sandboxDevShm := c.getSandboxDevShm(id) - if nsOptions.GetIpc() == runtime.NamespaceMode_NODE { - sandboxDevShm = devShm - } - // Remove the default /dev/shm mount from defaultMounts, it is added in oci/mounts.go. - specOpts = append(specOpts, oci.WithoutMounts(devShm)) - // In future the when user-namespace is enabled, the `nosuid, nodev, noexec` flags are - // required, otherwise the remount will fail with EPERM. Just use them unconditionally, - // they are nice to have anyways. - specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{ - { - Source: sandboxDevShm, - Destination: devShm, - Type: "bind", - Options: []string{"rbind", "ro", "nosuid", "nodev", "noexec"}, - }, - // Add resolv.conf for katacontainers to setup the DNS of pod VM properly. - { - Source: c.getResolvPath(id), - Destination: resolvConfPath, - Type: "bind", - Options: []string{"rbind", "ro"}, - }, - })) - - processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions()) - if err != nil { - return nil, fmt.Errorf("failed to init selinux options %+v: %w", securityContext.GetSelinuxOptions(), err) - } - defer func() { - if retErr != nil { - selinux.ReleaseLabel(processLabel) - } - }() - - supplementalGroups := securityContext.GetSupplementalGroups() - specOpts = append(specOpts, - customopts.WithSelinuxLabels(processLabel, mountLabel), - customopts.WithSupplementalGroups(supplementalGroups), - ) - - // Add sysctls - sysctls := config.GetLinux().GetSysctls() - if sysctls == nil { - sysctls = make(map[string]string) - } - _, ipUnprivilegedPortStart := sysctls["net.ipv4.ip_unprivileged_port_start"] - _, pingGroupRange := sysctls["net.ipv4.ping_group_range"] - if nsOptions.GetNetwork() != runtime.NamespaceMode_NODE { - if c.config.EnableUnprivilegedPorts && !ipUnprivilegedPortStart { - sysctls["net.ipv4.ip_unprivileged_port_start"] = "0" - } - if c.config.EnableUnprivilegedICMP && !pingGroupRange && !userns.RunningInUserNS() { - sysctls["net.ipv4.ping_group_range"] = "0 2147483647" - } - } - specOpts = append(specOpts, customopts.WithSysctls(sysctls)) - - // Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile - - if !c.config.DisableCgroup { - specOpts = append(specOpts, customopts.WithDefaultSandboxShares) - } - - if res := config.GetLinux().GetResources(); res != nil { - specOpts = append(specOpts, - customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)), - customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)), - customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)), - customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10))) - } - - specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj)) - - for pKey, pValue := range getPassthroughAnnotations(config.Annotations, - runtimePodAnnotations) { - specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue)) - } - - specOpts = append(specOpts, - customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox), - customopts.WithAnnotation(annotations.SandboxID, id), - customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()), - customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()), - customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()), - ) - - return c.runtimeSpec(id, "", specOpts...) -} - -// sandboxContainerSpecOpts generates OCI spec options for -// the sandbox container. -func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) { - var ( - securityContext = config.GetLinux().GetSecurityContext() - specOpts []oci.SpecOpts - err error - ) - ssp := securityContext.GetSeccomp() - if ssp == nil { - ssp, err = generateSeccompSecurityProfile( - securityContext.GetSeccompProfilePath(), //nolint:staticcheck // Deprecated but we don't want to remove yet - c.config.UnsetSeccompProfile) - if err != nil { - return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err) - } - } - seccompSpecOpts, err := c.generateSeccompSpecOpts( - ssp, - securityContext.GetPrivileged(), - c.seccompEnabled()) - if err != nil { - return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err) - } - if seccompSpecOpts != nil { - specOpts = append(specOpts, seccompSpecOpts) - } - - userstr, err := generateUserString( - "", - securityContext.GetRunAsUser(), - securityContext.GetRunAsGroup(), - ) - if err != nil { - return nil, fmt.Errorf("failed to generate user string: %w", err) - } - if userstr == "" { - // Lastly, since no user override was passed via CRI try to set via OCI - // Image - userstr = imageConfig.User - } - if userstr != "" { - specOpts = append(specOpts, oci.WithUser(userstr)) - } - return specOpts, nil -} - -// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts, -// /etc/resolv.conf and /etc/hostname. -func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { - sandboxEtcHostname := c.getSandboxHostname(id) - hostname := config.GetHostname() - if hostname == "" { - var err error - hostname, err = c.os.Hostname() - if err != nil { - return fmt.Errorf("failed to get hostname: %w", err) - } - } - if err := c.os.WriteFile(sandboxEtcHostname, []byte(hostname+"\n"), 0644); err != nil { - return fmt.Errorf("failed to write hostname to %q: %w", sandboxEtcHostname, err) - } - - // TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet. - sandboxEtcHosts := c.getSandboxHosts(id) - if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil { - return fmt.Errorf("failed to generate sandbox hosts file %q: %w", sandboxEtcHosts, err) - } - - // Set DNS options. Maintain a resolv.conf for the sandbox. - var err error - resolvContent := "" - if dnsConfig := config.GetDnsConfig(); dnsConfig != nil { - resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options) - if err != nil { - return fmt.Errorf("failed to parse sandbox DNSConfig %+v: %w", dnsConfig, err) - } - } - resolvPath := c.getResolvPath(id) - if resolvContent == "" { - // copy host's resolv.conf to resolvPath - err = c.os.CopyFile(resolvConfPath, resolvPath, 0644) - if err != nil { - return fmt.Errorf("failed to copy host's resolv.conf to %q: %w", resolvPath, err) - } - } else { - err = c.os.WriteFile(resolvPath, []byte(resolvContent), 0644) - if err != nil { - return fmt.Errorf("failed to write resolv content to %q: %w", resolvPath, err) - } - } - - // Setup sandbox /dev/shm. - if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE { - if _, err := c.os.Stat(devShm); err != nil { - return fmt.Errorf("host %q is not available for host ipc: %w", devShm, err) - } - } else { - sandboxDevShm := c.getSandboxDevShm(id) - if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil { - return fmt.Errorf("failed to create sandbox shm: %w", err) - } - shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize) - if err := c.os.(osinterface.UNIX).Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil { - return fmt.Errorf("failed to mount sandbox shm: %w", err) - } - } - - return nil -} - -// parseDNSOptions parse DNS options into resolv.conf format content, -// if none option is specified, will return empty with no error. -func parseDNSOptions(servers, searches, options []string) (string, error) { - resolvContent := "" - - if len(searches) > 0 { - resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " ")) - } - - if len(servers) > 0 { - resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver ")) - } - - if len(options) > 0 { - resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " ")) - } - - return resolvContent, nil -} - // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to // remove these files. Unmount should *NOT* return error if the mount point is already unmounted. func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { diff --git a/pkg/cri/sbserver/sandbox_run_linux_test.go b/pkg/cri/sbserver/sandbox_run_linux_test.go deleted file mode 100644 index 7059639c7..000000000 --- a/pkg/cri/sbserver/sandbox_run_linux_test.go +++ /dev/null @@ -1,526 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "os" - "path/filepath" - "strconv" - "testing" - - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/opencontainers/selinux/go-selinux" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - v1 "k8s.io/cri-api/pkg/apis/runtime/v1" - - "github.com/containerd/containerd/pkg/cri/annotations" - "github.com/containerd/containerd/pkg/cri/opts" - ostesting "github.com/containerd/containerd/pkg/os/testing" -) - -func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { - config := &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{ - Name: "test-name", - Uid: "test-uid", - Namespace: "test-ns", - Attempt: 1, - }, - Hostname: "test-hostname", - LogDirectory: "test-log-directory", - Labels: map[string]string{"a": "b"}, - Annotations: map[string]string{"c": "d"}, - Linux: &runtime.LinuxPodSandboxConfig{ - CgroupParent: "/test/cgroup/parent", - }, - } - imageConfig := &imagespec.ImageConfig{ - Env: []string{"a=b", "c=d"}, - Entrypoint: []string{"/pause"}, - Cmd: []string{"forever"}, - WorkingDir: "/workspace", - } - specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) { - assert.Equal(t, "test-hostname", spec.Hostname) - assert.Equal(t, getCgroupsPath("/test/cgroup/parent", id), spec.Linux.CgroupsPath) - assert.Equal(t, relativeRootfsPath, spec.Root.Path) - assert.Equal(t, true, spec.Root.Readonly) - assert.Contains(t, spec.Process.Env, "a=b", "c=d") - assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args) - assert.Equal(t, "/workspace", spec.Process.Cwd) - assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, opts.DefaultSandboxCPUshares) - assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj) - - t.Logf("Check PodSandbox annotations") - assert.Contains(t, spec.Annotations, annotations.SandboxID) - assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id) - - assert.Contains(t, spec.Annotations, annotations.ContainerType) - assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox) - - assert.Contains(t, spec.Annotations, annotations.SandboxNamespace) - assert.EqualValues(t, spec.Annotations[annotations.SandboxNamespace], "test-ns") - - assert.Contains(t, spec.Annotations, annotations.SandboxName) - assert.EqualValues(t, spec.Annotations[annotations.SandboxName], "test-name") - - assert.Contains(t, spec.Annotations, annotations.SandboxLogDir) - assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory") - - if selinux.GetEnabled() { - assert.NotEqual(t, "", spec.Process.SelinuxLabel) - assert.NotEqual(t, "", spec.Linux.MountLabel) - } - } - return config, imageConfig, specCheck -} - -func TestLinuxSandboxContainerSpec(t *testing.T) { - testID := "test-id" - nsPath := "test-cni" - for desc, test := range map[string]struct { - configChange func(*runtime.PodSandboxConfig) - specCheck func(*testing.T, *runtimespec.Spec) - expectErr bool - }{ - "spec should reflect original config": { - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - // runtime spec should have expected namespaces enabled by default. - require.NotNil(t, spec.Linux) - assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.NetworkNamespace, - Path: nsPath, - }) - assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.UTSNamespace, - }) - assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.PIDNamespace, - }) - assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.IPCNamespace, - }) - assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "0") - assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647") - }, - }, - "host namespace": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - } - }, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - // runtime spec should disable expected namespaces in host mode. - require.NotNil(t, spec.Linux) - assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.NetworkNamespace, - }) - assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.UTSNamespace, - }) - assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.PIDNamespace, - }) - assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{ - Type: runtimespec.IPCNamespace, - }) - assert.NotContains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "0") - assert.NotContains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647") - }, - }, - "should set supplemental groups correctly": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ - SupplementalGroups: []int64{1111, 2222}, - } - }, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - require.NotNil(t, spec.Process) - assert.Contains(t, spec.Process.User.AdditionalGids, uint32(1111)) - assert.Contains(t, spec.Process.User.AdditionalGids, uint32(2222)) - }, - }, - "should overwrite default sysctls": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.Sysctls = map[string]string{ - "net.ipv4.ip_unprivileged_port_start": "500", - "net.ipv4.ping_group_range": "1 1000", - } - }, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - require.NotNil(t, spec.Process) - assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "500") - assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "1 1000") - }, - }, - "sandbox sizing annotations should be set if LinuxContainerResources were provided": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.Resources = &v1.LinuxContainerResources{ - CpuPeriod: 100, - CpuQuota: 200, - CpuShares: 5000, - MemoryLimitInBytes: 1024, - } - }, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - value, ok := spec.Annotations[annotations.SandboxCPUPeriod] - assert.True(t, ok) - assert.EqualValues(t, strconv.FormatInt(100, 10), value) - assert.EqualValues(t, "100", value) - - value, ok = spec.Annotations[annotations.SandboxCPUQuota] - assert.True(t, ok) - assert.EqualValues(t, "200", value) - - value, ok = spec.Annotations[annotations.SandboxCPUShares] - assert.True(t, ok) - assert.EqualValues(t, "5000", value) - - value, ok = spec.Annotations[annotations.SandboxMem] - assert.True(t, ok) - assert.EqualValues(t, "1024", value) - }, - }, - "sandbox sizing annotations should not be set if LinuxContainerResources were not provided": { - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - _, ok := spec.Annotations[annotations.SandboxCPUPeriod] - assert.False(t, ok) - _, ok = spec.Annotations[annotations.SandboxCPUQuota] - assert.False(t, ok) - _, ok = spec.Annotations[annotations.SandboxCPUShares] - assert.False(t, ok) - _, ok = spec.Annotations[annotations.SandboxMem] - assert.False(t, ok) - }, - }, - "sandbox sizing annotations are zero if the resources are set to 0": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Linux.Resources = &v1.LinuxContainerResources{} - }, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - value, ok := spec.Annotations[annotations.SandboxCPUPeriod] - assert.True(t, ok) - assert.EqualValues(t, "0", value) - value, ok = spec.Annotations[annotations.SandboxCPUQuota] - assert.True(t, ok) - assert.EqualValues(t, "0", value) - value, ok = spec.Annotations[annotations.SandboxCPUShares] - assert.True(t, ok) - assert.EqualValues(t, "0", value) - value, ok = spec.Annotations[annotations.SandboxMem] - assert.True(t, ok) - assert.EqualValues(t, "0", value) - }, - }, - } { - t.Run(desc, func(t *testing.T) { - c := newTestCRIService() - c.config.EnableUnprivilegedICMP = true - c.config.EnableUnprivilegedPorts = true - config, imageConfig, specCheck := getRunPodSandboxTestData() - if test.configChange != nil { - test.configChange(config) - } - spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil) - if test.expectErr { - assert.Error(t, err) - assert.Nil(t, spec) - return - } - assert.NoError(t, err) - assert.NotNil(t, spec) - specCheck(t, testID, spec) - if test.specCheck != nil { - test.specCheck(t, spec) - } - }) - } -} - -func TestSetupSandboxFiles(t *testing.T) { - const ( - testID = "test-id" - realhostname = "test-real-hostname" - ) - for desc, test := range map[string]struct { - dnsConfig *runtime.DNSConfig - hostname string - ipcMode runtime.NamespaceMode - expectedCalls []ostesting.CalledDetail - }{ - "should check host /dev/shm existence when ipc mode is NODE": { - ipcMode: runtime.NamespaceMode_NODE, - expectedCalls: []ostesting.CalledDetail{ - { - Name: "Hostname", - }, - { - Name: "WriteFile", - Arguments: []interface{}{ - filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), - []byte(realhostname + "\n"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/hosts", - filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/resolv.conf", - filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), - os.FileMode(0644), - }, - }, - { - Name: "Stat", - Arguments: []interface{}{"/dev/shm"}, - }, - }, - }, - "should create new /etc/resolv.conf if DNSOptions is set": { - dnsConfig: &runtime.DNSConfig{ - Servers: []string{"8.8.8.8"}, - Searches: []string{"114.114.114.114"}, - Options: []string{"timeout:1"}, - }, - ipcMode: runtime.NamespaceMode_NODE, - expectedCalls: []ostesting.CalledDetail{ - { - Name: "Hostname", - }, - { - Name: "WriteFile", - Arguments: []interface{}{ - filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), - []byte(realhostname + "\n"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/hosts", - filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), - os.FileMode(0644), - }, - }, - { - Name: "WriteFile", - Arguments: []interface{}{ - filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), - []byte(`search 114.114.114.114 -nameserver 8.8.8.8 -options timeout:1 -`), os.FileMode(0644), - }, - }, - { - Name: "Stat", - Arguments: []interface{}{"/dev/shm"}, - }, - }, - }, - "should create sandbox shm when ipc namespace mode is not NODE": { - ipcMode: runtime.NamespaceMode_POD, - expectedCalls: []ostesting.CalledDetail{ - { - Name: "Hostname", - }, - { - Name: "WriteFile", - Arguments: []interface{}{ - filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), - []byte(realhostname + "\n"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/hosts", - filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/resolv.conf", - filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), - os.FileMode(0644), - }, - }, - { - Name: "MkdirAll", - Arguments: []interface{}{ - filepath.Join(testStateDir, sandboxesDir, testID, "shm"), - os.FileMode(0700), - }, - }, - { - Name: "Mount", - // Ignore arguments which are too complex to check. - }, - }, - }, - "should create /etc/hostname when hostname is set": { - hostname: "test-hostname", - ipcMode: runtime.NamespaceMode_NODE, - expectedCalls: []ostesting.CalledDetail{ - { - Name: "WriteFile", - Arguments: []interface{}{ - filepath.Join(testRootDir, sandboxesDir, testID, "hostname"), - []byte("test-hostname\n"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/hosts", - filepath.Join(testRootDir, sandboxesDir, testID, "hosts"), - os.FileMode(0644), - }, - }, - { - Name: "CopyFile", - Arguments: []interface{}{ - "/etc/resolv.conf", - filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"), - os.FileMode(0644), - }, - }, - { - Name: "Stat", - Arguments: []interface{}{"/dev/shm"}, - }, - }, - }, - } { - t.Run(desc, func(t *testing.T) { - c := newTestCRIService() - c.os.(*ostesting.FakeOS).HostnameFn = func() (string, error) { - return realhostname, nil - } - cfg := &runtime.PodSandboxConfig{ - Hostname: test.hostname, - DnsConfig: test.dnsConfig, - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - NamespaceOptions: &runtime.NamespaceOption{ - Ipc: test.ipcMode, - }, - }, - }, - } - c.setupSandboxFiles(testID, cfg) - calls := c.os.(*ostesting.FakeOS).GetCalls() - assert.Len(t, calls, len(test.expectedCalls)) - for i, expected := range test.expectedCalls { - if expected.Arguments == nil { - // Ignore arguments. - expected.Arguments = calls[i].Arguments - } - assert.Equal(t, expected, calls[i]) - } - }) - } -} - -func TestParseDNSOption(t *testing.T) { - for desc, test := range map[string]struct { - servers []string - searches []string - options []string - expectedContent string - expectErr bool - }{ - "empty dns options should return empty content": {}, - "non-empty dns options should return correct content": { - servers: []string{"8.8.8.8", "server.google.com"}, - searches: []string{"114.114.114.114"}, - options: []string{"timeout:1"}, - expectedContent: `search 114.114.114.114 -nameserver 8.8.8.8 -nameserver server.google.com -options timeout:1 -`, - }, - "expanded dns config should return correct content on modern libc (e.g. glibc 2.26 and above)": { - servers: []string{"8.8.8.8", "server.google.com"}, - searches: []string{ - "server0.google.com", - "server1.google.com", - "server2.google.com", - "server3.google.com", - "server4.google.com", - "server5.google.com", - "server6.google.com", - }, - options: []string{"timeout:1"}, - expectedContent: `search server0.google.com server1.google.com server2.google.com server3.google.com server4.google.com server5.google.com server6.google.com -nameserver 8.8.8.8 -nameserver server.google.com -options timeout:1 -`, - }, - } { - t.Run(desc, func(t *testing.T) { - resolvContent, err := parseDNSOptions(test.servers, test.searches, test.options) - if test.expectErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, resolvContent, test.expectedContent) - }) - } -} - -func TestSandboxDisableCgroup(t *testing.T) { - config, imageConfig, _ := getRunPodSandboxTestData() - c := newTestCRIService() - c.config.DisableCgroup = true - spec, err := c.sandboxContainerSpec("test-id", config, imageConfig, "test-cni", []string{}) - require.NoError(t, err) - - t.Log("resource limit should not be set") - assert.Nil(t, spec.Linux.Resources.Memory) - assert.Nil(t, spec.Linux.Resources.CPU) - - t.Log("cgroup path should be empty") - assert.Empty(t, spec.Linux.CgroupsPath) -} - -// TODO(random-liu): [P1] Add unit test for different error cases to make sure -// the function cleans up on error properly. diff --git a/pkg/cri/sbserver/sandbox_run_other.go b/pkg/cri/sbserver/sandbox_run_other.go index e3e0e53ed..26aaef540 100644 --- a/pkg/cri/sbserver/sandbox_run_other.go +++ b/pkg/cri/sbserver/sandbox_run_other.go @@ -21,29 +21,9 @@ package sbserver import ( "github.com/containerd/containerd" - "github.com/containerd/containerd/oci" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) -func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig, - imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) { - return c.runtimeSpec(id, "") -} - -// sandboxContainerSpecOpts generates OCI spec options for -// the sandbox container. -func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) { - return []oci.SpecOpts{}, nil -} - -// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts, -// /etc/resolv.conf and /etc/hostname. -func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { - return nil -} - // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to // remove these files. Unmount should *NOT* return error if the mount point is already unmounted. func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { diff --git a/pkg/cri/sbserver/sandbox_run_other_test.go b/pkg/cri/sbserver/sandbox_run_other_test.go deleted file mode 100644 index d420e672f..000000000 --- a/pkg/cri/sbserver/sandbox_run_other_test.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build !windows && !linux -// +build !windows,!linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "testing" - - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" -) - -func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { - config := &runtime.PodSandboxConfig{} - imageConfig := &imagespec.ImageConfig{} - specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) { - } - return config, imageConfig, specCheck -} diff --git a/pkg/cri/sbserver/sandbox_run_test.go b/pkg/cri/sbserver/sandbox_run_test.go index d92544865..a80f13d09 100644 --- a/pkg/cri/sbserver/sandbox_run_test.go +++ b/pkg/cri/sbserver/sandbox_run_test.go @@ -19,152 +19,13 @@ package sbserver import ( "context" "net" - goruntime "runtime" "testing" "github.com/containerd/go-cni" - "github.com/containerd/typeurl" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - "github.com/containerd/containerd/pkg/cri/annotations" - criconfig "github.com/containerd/containerd/pkg/cri/config" - sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox" ) -func TestSandboxContainerSpec(t *testing.T) { - switch goruntime.GOOS { - case "darwin": - t.Skip("not implemented on Darwin") - case "freebsd": - t.Skip("not implemented on FreeBSD") - } - testID := "test-id" - nsPath := "test-cni" - for desc, test := range map[string]struct { - configChange func(*runtime.PodSandboxConfig) - podAnnotations []string - imageConfigChange func(*imagespec.ImageConfig) - specCheck func(*testing.T, *runtimespec.Spec) - expectErr bool - }{ - "should return error when entrypoint and cmd are empty": { - imageConfigChange: func(c *imagespec.ImageConfig) { - c.Entrypoint = nil - c.Cmd = nil - }, - expectErr: true, - }, - "a passthrough annotation should be passed as an OCI annotation": { - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - }, - }, - "a non-passthrough annotation should not be passed as an OCI annotation": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["d"] = "e" - }, - podAnnotations: []string{"c"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["c"], "d") - _, ok := spec.Annotations["d"] - assert.False(t, ok) - }, - }, - "passthrough annotations should support wildcard match": { - configChange: func(c *runtime.PodSandboxConfig) { - c.Annotations["t.f"] = "j" - c.Annotations["z.g"] = "o" - c.Annotations["z"] = "o" - c.Annotations["y.ca"] = "b" - c.Annotations["y"] = "b" - }, - podAnnotations: []string{"t*", "z.*", "y.c*"}, - specCheck: func(t *testing.T, spec *runtimespec.Spec) { - assert.Equal(t, spec.Annotations["t.f"], "j") - assert.Equal(t, spec.Annotations["z.g"], "o") - assert.Equal(t, spec.Annotations["y.ca"], "b") - _, ok := spec.Annotations["y"] - assert.False(t, ok) - _, ok = spec.Annotations["z"] - assert.False(t, ok) - }, - }, - } { - t.Run(desc, func(t *testing.T) { - c := newTestCRIService() - config, imageConfig, specCheck := getRunPodSandboxTestData() - if test.configChange != nil { - test.configChange(config) - } - - if test.imageConfigChange != nil { - test.imageConfigChange(imageConfig) - } - spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, - test.podAnnotations) - if test.expectErr { - assert.Error(t, err) - assert.Nil(t, spec) - return - } - assert.NoError(t, err) - assert.NotNil(t, spec) - specCheck(t, testID, spec) - if test.specCheck != nil { - test.specCheck(t, spec) - } - }) - } -} - -func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) { - for desc, test := range map[string]struct { - configChange func(*runtime.PodSandboxConfig) - }{ - "should marshal original config": {}, - "should marshal Linux": { - configChange: func(c *runtime.PodSandboxConfig) { - if c.Linux == nil { - c.Linux = &runtime.LinuxPodSandboxConfig{} - } - c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{ - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - SupplementalGroups: []int64{1111, 2222}, - } - }, - }, - } { - t.Run(desc, func(t *testing.T) { - meta := &sandboxstore.Metadata{ - ID: "1", - Name: "sandbox_1", - NetNSPath: "/home/cloud", - } - meta.Config, _, _ = getRunPodSandboxTestData() - if test.configChange != nil { - test.configChange(meta.Config) - } - - any, err := typeurl.MarshalAny(meta) - assert.NoError(t, err) - data, err := typeurl.UnmarshalAny(any) - assert.NoError(t, err) - assert.IsType(t, &sandboxstore.Metadata{}, data) - curMeta, ok := data.(*sandboxstore.Metadata) - assert.True(t, ok) - assert.Equal(t, meta, curMeta) - }) - } -} - func TestToCNIPortMappings(t *testing.T) { for desc, test := range map[string]struct { criPortMappings []*runtime.PortMapping @@ -315,212 +176,3 @@ func TestSelectPodIP(t *testing.T) { }) } } - -func TestHostAccessingSandbox(t *testing.T) { - privilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - }, - } - nonPrivilegedContext := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - }, - }, - } - hostNamespace := &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - }, - }, - } - tests := []struct { - name string - config *runtime.PodSandboxConfig - want bool - }{ - {"Security Context is nil", nil, false}, - {"Security Context is privileged", privilegedContext, false}, - {"Security Context is not privileged", nonPrivilegedContext, false}, - {"Security Context namespace host access", hostNamespace, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := hostAccessingSandbox(tt.config); got != tt.want { - t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetSandboxRuntime(t *testing.T) { - untrustedWorkloadRuntime := criconfig.Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "untrusted-workload-runtime", - Root: "", - } - - defaultRuntime := criconfig.Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "default-runtime", - Root: "", - } - - fooRuntime := criconfig.Runtime{ - Type: "io.containerd.runtime.v1.linux", - Engine: "foo-bar", - Root: "", - } - - for desc, test := range map[string]struct { - sandboxConfig *runtime.PodSandboxConfig - runtimeHandler string - runtimes map[string]criconfig.Runtime - expectErr bool - expectedRuntime criconfig.Runtime - }{ - "should return error if untrusted workload requires host access": { - sandboxConfig: &runtime.PodSandboxConfig{ - Linux: &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: false, - NamespaceOptions: &runtime.NamespaceOption{ - Network: runtime.NamespaceMode_NODE, - Pid: runtime.NamespaceMode_NODE, - Ipc: runtime.NamespaceMode_NODE, - }, - }, - }, - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - }, - expectErr: true, - }, - "should use untrusted workload runtime for untrusted workload": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - }, - expectedRuntime: untrustedWorkloadRuntime, - }, - "should use default runtime for regular workload": { - sandboxConfig: &runtime.PodSandboxConfig{}, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - }, - expectedRuntime: defaultRuntime, - }, - "should use default runtime for trusted workload": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "false", - }, - }, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - }, - expectedRuntime: defaultRuntime, - }, - "should return error if untrusted workload runtime is required but not configured": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - }, - expectErr: true, - }, - "should use 'untrusted' runtime for untrusted workload": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - }, - expectedRuntime: untrustedWorkloadRuntime, - }, - "should use 'untrusted' runtime for untrusted workload & handler": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimeHandler: "untrusted", - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - }, - expectedRuntime: untrustedWorkloadRuntime, - }, - "should return an error if untrusted annotation with conflicting handler": { - sandboxConfig: &runtime.PodSandboxConfig{ - Annotations: map[string]string{ - annotations.UntrustedWorkload: "true", - }, - }, - runtimeHandler: "foo", - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - "foo": fooRuntime, - }, - expectErr: true, - }, - "should use correct runtime for a runtime handler": { - sandboxConfig: &runtime.PodSandboxConfig{}, - runtimeHandler: "foo", - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - criconfig.RuntimeUntrusted: untrustedWorkloadRuntime, - "foo": fooRuntime, - }, - expectedRuntime: fooRuntime, - }, - "should return error if runtime handler is required but not configured": { - sandboxConfig: &runtime.PodSandboxConfig{}, - runtimeHandler: "bar", - runtimes: map[string]criconfig.Runtime{ - criconfig.RuntimeDefault: defaultRuntime, - "foo": fooRuntime, - }, - expectErr: true, - }, - } { - t.Run(desc, func(t *testing.T) { - cri := newTestCRIService() - cri.config = criconfig.Config{ - PluginConfig: criconfig.DefaultConfig(), - } - cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault - cri.config.ContainerdConfig.Runtimes = test.runtimes - r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler) - assert.Equal(t, test.expectErr, err != nil) - assert.Equal(t, test.expectedRuntime, r) - }) - } -} diff --git a/pkg/cri/sbserver/sandbox_run_windows.go b/pkg/cri/sbserver/sandbox_run_windows.go index db363c434..37abe22ae 100644 --- a/pkg/cri/sbserver/sandbox_run_windows.go +++ b/pkg/cri/sbserver/sandbox_run_windows.go @@ -17,91 +17,10 @@ package sbserver import ( - "fmt" - "strconv" - "github.com/containerd/containerd" - "github.com/containerd/containerd/oci" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - "github.com/containerd/containerd/pkg/cri/annotations" - customopts "github.com/containerd/containerd/pkg/cri/opts" ) -func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig, - imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) { - // Creates a spec Generator with the default spec. - specOpts := []oci.SpecOpts{ - oci.WithEnv(imageConfig.Env), - oci.WithHostname(config.GetHostname()), - } - if imageConfig.WorkingDir != "" { - specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir)) - } - - if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 { - // Pause image must have entrypoint or cmd. - return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig) - } - specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...)) - - specOpts = append(specOpts, - // Clear the root location since hcsshim expects it. - // NOTE: readonly rootfs doesn't work on windows. - customopts.WithoutRoot, - customopts.WithWindowsNetworkNamespace(nsPath), - ) - - specOpts = append(specOpts, customopts.WithWindowsDefaultSandboxShares) - - // Start with the image config user and override below if RunAsUsername is not "". - username := imageConfig.User - - runAsUser := config.GetWindows().GetSecurityContext().GetRunAsUsername() - if runAsUser != "" { - username = runAsUser - } - - cs := config.GetWindows().GetSecurityContext().GetCredentialSpec() - if cs != "" { - specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs)) - } - - // There really isn't a good Windows way to verify that the username is available in the - // image as early as here like there is for Linux. Later on in the stack hcsshim - // will handle the behavior of erroring out if the user isn't available in the image - // when trying to run the init process. - specOpts = append(specOpts, oci.WithUser(username)) - - for pKey, pValue := range getPassthroughAnnotations(config.Annotations, - runtimePodAnnotations) { - specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue)) - } - - specOpts = append(specOpts, - customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox), - customopts.WithAnnotation(annotations.SandboxID, id), - customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()), - customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()), - customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()), - customopts.WithAnnotation(annotations.WindowsHostProcess, strconv.FormatBool(config.GetWindows().GetSecurityContext().GetHostProcess())), - ) - - return c.runtimeSpec(id, "", specOpts...) -} - -// No sandbox container spec options for windows yet. -func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) { - return nil, nil -} - -// No sandbox files needed for windows. -func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { - return nil -} - // No sandbox files needed for windows. func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { return nil diff --git a/pkg/cri/sbserver/sandbox_run_windows_test.go b/pkg/cri/sbserver/sandbox_run_windows_test.go deleted file mode 100644 index 6569f67e0..000000000 --- a/pkg/cri/sbserver/sandbox_run_windows_test.go +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "testing" - - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/stretchr/testify/assert" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - "github.com/containerd/containerd/pkg/cri/annotations" - "github.com/containerd/containerd/pkg/cri/opts" -) - -func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) { - config := &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{ - Name: "test-name", - Uid: "test-uid", - Namespace: "test-ns", - Attempt: 1, - }, - Hostname: "test-hostname", - LogDirectory: "test-log-directory", - Labels: map[string]string{"a": "b"}, - Annotations: map[string]string{"c": "d"}, - Windows: &runtime.WindowsPodSandboxConfig{ - SecurityContext: &runtime.WindowsSandboxSecurityContext{ - RunAsUsername: "test-user", - CredentialSpec: "{\"test\": \"spec\"}", - HostProcess: false, - }, - }, - } - imageConfig := &imagespec.ImageConfig{ - Env: []string{"a=b", "c=d"}, - Entrypoint: []string{"/pause"}, - Cmd: []string{"forever"}, - WorkingDir: "/workspace", - User: "test-image-user", - } - specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) { - assert.Equal(t, "test-hostname", spec.Hostname) - assert.Nil(t, spec.Root) - assert.Contains(t, spec.Process.Env, "a=b", "c=d") - assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args) - assert.Equal(t, "/workspace", spec.Process.Cwd) - assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, opts.DefaultSandboxCPUshares) - - // Also checks if override of the image configs user is behaving. - t.Logf("Check username") - assert.Contains(t, spec.Process.User.Username, "test-user") - - t.Logf("Check credential spec") - assert.Contains(t, spec.Windows.CredentialSpec, "{\"test\": \"spec\"}") - - t.Logf("Check PodSandbox annotations") - assert.Contains(t, spec.Annotations, annotations.SandboxID) - assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id) - - assert.Contains(t, spec.Annotations, annotations.ContainerType) - assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox) - - assert.Contains(t, spec.Annotations, annotations.SandboxNamespace) - assert.EqualValues(t, spec.Annotations[annotations.SandboxNamespace], "test-ns") - - assert.Contains(t, spec.Annotations, annotations.SandboxName) - assert.EqualValues(t, spec.Annotations[annotations.SandboxName], "test-name") - - assert.Contains(t, spec.Annotations, annotations.SandboxLogDir) - assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory") - - assert.Contains(t, spec.Annotations, annotations.WindowsHostProcess) - assert.EqualValues(t, spec.Annotations[annotations.WindowsHostProcess], "false") - } - return config, imageConfig, specCheck -} - -func TestSandboxWindowsNetworkNamespace(t *testing.T) { - testID := "test-id" - nsPath := "test-cni" - c := newTestCRIService() - - config, imageConfig, specCheck := getRunPodSandboxTestData() - spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil) - assert.NoError(t, err) - assert.NotNil(t, spec) - specCheck(t, testID, spec) - assert.NotNil(t, spec.Windows) - assert.NotNil(t, spec.Windows.Network) - assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace) -}