github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Use container'd oci opts for spec generation

Browse Source 
This bumps the containerd and sys packages in CRI

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Remove runtime-tools

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update tests for oci opts package

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby
2019-03-15 15:36:06 -04:00
parent eb27e526f5
commit 5eddc1a2cc
356 changed files with 59642 additions and 20733 deletions
Download Patch File Download Diff File Expand all files Collapse all files

602
pkg/server/container_create.go
View File

@@ -17,9 +17,7 @@ limitations under the License.
package server
import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
@@ -28,20 +26,14 @@ import (
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/contrib/apparmor"
"github.com/containerd/containerd/contrib/seccomp"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/oci"
"github.com/containerd/typeurl"
"github.com/davecgh/go-spew/spew"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/runc/libcontainer/devices"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/validate"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/syndtr/gocapability/capability"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"github.com/containerd/cri/pkg/annotations"
@@ -317,52 +309,45 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
}
func (c *criService) generateContainerSpec(id string, sandboxID string, sandboxPid uint32, config *runtime.ContainerConfig,
sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, extraMounts []*runtime.Mount,
runtimePodAnnotations []string) (*runtimespec.Spec, error) {
// Creates a spec Generator with the default spec.
spec, err := defaultRuntimeSpec(id)
if err != nil {
return nil, err
sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, extraMounts []*runtime.Mount, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
specOpts := []oci.SpecOpts{
customopts.WithoutRunMount,
customopts.WithoutDefaultSecuritySettings,
customopts.WithRelativeRoot(relativeRootfsPath),
customopts.WithProcessArgs(config, imageConfig),
// this will be set based on the security context below
oci.WithNewPrivileges,
}
g := newSpecGenerator(spec)
// Set the relative path to the rootfs of the container from containerd's
// pre-defined directory.
g.SetRootPath(relativeRootfsPath)
if err := setOCIProcessArgs(&g, config, imageConfig); err != nil {
return nil, err
}
if config.GetWorkingDir() != "" {
g.SetProcessCwd(config.GetWorkingDir())
specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir()))
} else if imageConfig.WorkingDir != "" {
g.SetProcessCwd(imageConfig.WorkingDir)
specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
}
g.SetProcessTerminal(config.GetTty())
if config.GetTty() {
g.AddProcessEnv("TERM", "xterm")
specOpts = append(specOpts, oci.WithTTY)
}
// Add HOSTNAME env.
hostname := sandboxConfig.GetHostname()
if sandboxConfig.GetHostname() == "" {
hostname, err = c.os.Hostname()
if err != nil {
var (
err error
hostname = sandboxConfig.GetHostname()
)
if hostname == "" {
if hostname, err = c.os.Hostname(); err != nil {
return nil, err
}
}
g.AddProcessEnv(hostnameEnv, hostname)
specOpts = append(specOpts, oci.WithEnv([]string{hostnameEnv + "=" + hostname}))
// Apply envs from image config first, so that envs from container config
// can override them.
if err := addImageEnvs(&g, imageConfig.Env); err != nil {
return nil, err
}
env := imageConfig.Env
for _, e := range config.GetEnvs() {
g.AddProcessEnv(e.GetKey(), e.GetValue())
env = append(env, e.GetKey()+"="+e.GetValue())
}
specOpts = append(specOpts, oci.WithEnv(env))
securityContext := config.GetLinux().GetSecurityContext()
selinuxOpt := securityContext.GetSelinuxOptions()
@@ -370,97 +355,78 @@ func (c *criService) generateContainerSpec(id string, sandboxID string, sandboxP
if err != nil {
return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
}
// Merge extra mounts and CRI mounts.
mounts := mergeMounts(config.GetMounts(), extraMounts)
if err := c.addOCIBindMounts(&g, mounts, mountLabel); err != nil {
return nil, errors.Wrapf(err, "failed to set OCI bind mounts %+v", mounts)
}
specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel))
// Apply masked paths if specified.
// When `MaskedPaths` is not specified, keep runtime default for backward compatibility;
// When `MaskedPaths` is specified, but length is zero, clear masked path list.
// Note: If the container is privileged, then we clear any masked paths later on in the call to setOCIPrivileged()
if securityContext.GetMaskedPaths() != nil {
g.Config.Linux.MaskedPaths = nil
for _, path := range securityContext.GetMaskedPaths() {
g.AddLinuxMaskedPaths(path)
}
if maskedPaths := securityContext.GetMaskedPaths(); maskedPaths != nil {
specOpts = append(specOpts, oci.WithMaskedPaths(maskedPaths))
}
// Apply readonly paths if specified.
// Note: If the container is privileged, then we clear any readonly paths later on in the call to setOCIPrivileged()
if securityContext.GetReadonlyPaths() != nil {
g.Config.Linux.ReadonlyPaths = nil
for _, path := range securityContext.GetReadonlyPaths() {
g.AddLinuxReadonlyPaths(path)
}
// Apply readonly paths if specified.
if roPaths := securityContext.GetReadonlyPaths(); roPaths != nil {
specOpts = append(specOpts, oci.WithReadonlyPaths(roPaths))
}
if securityContext.GetPrivileged() {
if !sandboxConfig.GetLinux().GetSecurityContext().GetPrivileged() {
return nil, errors.New("no privileged container allowed in sandbox")
}
if err := setOCIPrivileged(&g, config); err != nil {
return nil, err
}
specOpts = append(specOpts, oci.WithPrivileged, customopts.WithPrivilegedDevices)
} else { // not privileged
if err := c.addOCIDevices(&g, config.GetDevices()); err != nil {
return nil, errors.Wrapf(err, "failed to set devices mapping %+v", config.GetDevices())
}
if err := setOCICapabilities(&g, securityContext.GetCapabilities()); err != nil {
return nil, errors.Wrapf(err, "failed to set capabilities %+v",
securityContext.GetCapabilities())
}
specOpts = append(specOpts, customopts.WithDevices(c.os, config), customopts.WithCapabilities(securityContext))
}
// Clear all ambient capabilities. The implication of non-root + caps
// is not clearly defined in Kubernetes.
// See https://github.com/kubernetes/kubernetes/issues/56374
// Keep docker's behavior for now.
g.Config.Process.Capabilities.Ambient = []string{}
g.SetProcessSelinuxLabel(processLabel)
g.SetLinuxMountLabel(mountLabel)
specOpts = append(specOpts,
customopts.WithoutAmbientCaps,
customopts.WithSelinuxLabels(processLabel, mountLabel),
)
// TODO: Figure out whether we should set no new privilege for sandbox container by default
g.SetProcessNoNewPrivileges(securityContext.GetNoNewPrivs())
if securityContext.GetNoNewPrivs() {
specOpts = append(specOpts, oci.WithNoNewPrivileges)
}
// TODO(random-liu): [P1] Set selinux options (privileged or not).
g.SetRootReadonly(securityContext.GetReadonlyRootfs())
if securityContext.GetReadonlyRootfs() {
specOpts = append(specOpts, oci.WithRootFSReadonly())
}
if c.config.DisableCgroup {
g.SetLinuxCgroupsPath("")
specOpts = append(specOpts, customopts.WithDisabledCgroups)
} else {
setOCILinuxResourceCgroup(&g, config.GetLinux().GetResources())
specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources()))
if sandboxConfig.GetLinux().GetCgroupParent() != "" {
cgroupsPath := getCgroupsPath(sandboxConfig.GetLinux().GetCgroupParent(), id,
c.config.SystemdCgroup)
g.SetLinuxCgroupsPath(cgroupsPath)
specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
}
}
if err := setOCILinuxResourceOOMScoreAdj(&g, config.GetLinux().GetResources(), c.config.RestrictOOMScoreAdj); err != nil {
return nil, err
}
// Set namespaces, share namespace with sandbox container.
setOCINamespaces(&g, securityContext.GetNamespaceOptions(), sandboxPid)
supplementalGroups := securityContext.GetSupplementalGroups()
for _, group := range supplementalGroups {
g.AddProcessAdditionalGid(uint32(group))
}
for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations,
runtimePodAnnotations) {
g.AddAnnotation(pKey, pValue)
specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
}
g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer)
g.AddAnnotation(annotations.SandboxID, sandboxID)
specOpts = append(specOpts,
customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj),
customopts.WithPodNamespaces(securityContext, sandboxPid),
customopts.WithSupplementalGroups(supplementalGroups),
customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer),
customopts.WithAnnotation(annotations.SandboxID, sandboxID),
)
return g.Config, nil
return runtimeSpec(id, specOpts...)
}
// generateVolumeMounts sets up image volumes for container. Rely on the removal of container
@@ -545,410 +511,14 @@ func (c *criService) generateContainerMounts(sandboxID string, config *runtime.C
return mounts
}
// setOCIProcessArgs sets process args. It returns error if the final arg list
// is empty.
func setOCIProcessArgs(g *generator, config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) error {
command, args := config.GetCommand(), config.GetArgs()
// The following logic is migrated from https://github.com/moby/moby/blob/master/daemon/commit.go
// TODO(random-liu): Clearly define the commands overwrite behavior.
if len(command) == 0 {
// Copy array to avoid data race.
if len(args) == 0 {
args = append([]string{}, imageConfig.Cmd...)
}
if command == nil {
command = append([]string{}, imageConfig.Entrypoint...)
}
}
if len(command) == 0 && len(args) == 0 {
return errors.New("no command specified")
}
g.SetProcessArgs(append(command, args...))
return nil
}
// addImageEnvs adds environment variables from image config. It returns error if
// an invalid environment variable is encountered.
func addImageEnvs(g *generator, imageEnvs []string) error {
for _, e := range imageEnvs {
kv := strings.SplitN(e, "=", 2)
if len(kv) != 2 {
return errors.Errorf("invalid environment variable %q", e)
}
g.AddProcessEnv(kv[0], kv[1])
}
return nil
}
func setOCIPrivileged(g *generator, config *runtime.ContainerConfig) error {
// Add all capabilities in privileged mode.
g.SetupPrivileged(true)
setOCIBindMountsPrivileged(g)
if err := setOCIDevicesPrivileged(g); err != nil {
return errors.Wrapf(err, "failed to set devices mapping %+v", config.GetDevices())
}
return nil
}
func clearReadOnly(m *runtimespec.Mount) {
var opt []string
for _, o := range m.Options {
if o != "ro" {
opt = append(opt, o)
}
}
m.Options = append(opt, "rw")
}
// addDevices set device mapping without privilege.
func (c *criService) addOCIDevices(g *generator, devs []*runtime.Device) error {
spec := g.Config
for _, device := range devs {
path, err := c.os.ResolveSymbolicLink(device.HostPath)
if err != nil {
return err
}
dev, err := devices.DeviceFromPath(path, device.Permissions)
if err != nil {
return err
}
rd := runtimespec.LinuxDevice{
Path: device.ContainerPath,
Type: string(dev.Type),
Major: dev.Major,
Minor: dev.Minor,
UID: &dev.Uid,
GID: &dev.Gid,
}
g.AddDevice(rd)
spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, runtimespec.LinuxDeviceCgroup{
Allow: true,
Type: string(dev.Type),
Major: &dev.Major,
Minor: &dev.Minor,
Access: dev.Permissions,
})
}
return nil
}
// addDevices set device mapping with privilege.
func setOCIDevicesPrivileged(g *generator) error {
spec := g.Config
hostDevices, err := devices.HostDevices()
if err != nil {
return err
}
for _, hostDevice := range hostDevices {
rd := runtimespec.LinuxDevice{
Path: hostDevice.Path,
Type: string(hostDevice.Type),
Major: hostDevice.Major,
Minor: hostDevice.Minor,
UID: &hostDevice.Uid,
GID: &hostDevice.Gid,
}
if hostDevice.Major == 0 && hostDevice.Minor == 0 {
// Invalid device, most likely a symbolic link, skip it.
continue
}
g.AddDevice(rd)
}
spec.Linux.Resources.Devices = []runtimespec.LinuxDeviceCgroup{
{
Allow: true,
Access: "rwm",
},
}
return nil
}
// addOCIBindMounts adds bind mounts.
func (c *criService) addOCIBindMounts(g *generator, mounts []*runtime.Mount, mountLabel string) error {
// Sort mounts in number of parts. This ensures that high level mounts don't
// shadow other mounts.
sort.Sort(orderedMounts(mounts))
// Mount cgroup into the container as readonly, which inherits docker's behavior.
g.AddMount(runtimespec.Mount{
Source: "cgroup",
Destination: "/sys/fs/cgroup",
Type: "cgroup",
Options: []string{"nosuid", "noexec", "nodev", "relatime", "ro"},
})
// Copy all mounts from default mounts, except for
// - mounts overriden by supplied mount;
// - all mounts under /dev if a supplied /dev is present.
mountSet := make(map[string]struct{})
for _, m := range mounts {
mountSet[filepath.Clean(m.ContainerPath)] = struct{}{}
}
defaultMounts := g.Mounts()
g.ClearMounts()
for _, m := range defaultMounts {
dst := filepath.Clean(m.Destination)
if _, ok := mountSet[dst]; ok {
// filter out mount overridden by a supplied mount
continue
}
if _, mountDev := mountSet["/dev"]; mountDev && strings.HasPrefix(dst, "/dev/") {
// filter out everything under /dev if /dev is a supplied mount
continue
}
g.AddMount(m)
}
for _, mount := range mounts {
dst := mount.GetContainerPath()
src := mount.GetHostPath()
// Create the host path if it doesn't exist.
// TODO(random-liu): Add CRI validation test for this case.
if _, err := c.os.Stat(src); err != nil {
if !os.IsNotExist(err) {
return errors.Wrapf(err, "failed to stat %q", src)
}
if err := c.os.MkdirAll(src, 0755); err != nil {
return errors.Wrapf(err, "failed to mkdir %q", src)
}
}
// TODO(random-liu): Add cri-containerd integration test or cri validation test
// for this.
src, err := c.os.ResolveSymbolicLink(src)
if err != nil {
return errors.Wrapf(err, "failed to resolve symlink %q", src)
}
options := []string{"rbind"}
switch mount.GetPropagation() {
case runtime.MountPropagation_PROPAGATION_PRIVATE:
options = append(options, "rprivate")
// Since default root propogation in runc is rprivate ignore
// setting the root propagation
case runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL:
if err := ensureShared(src, c.os.LookupMount); err != nil {
return err
}
options = append(options, "rshared")
g.SetLinuxRootPropagation("rshared") // nolint: errcheck
case runtime.MountPropagation_PROPAGATION_HOST_TO_CONTAINER:
if err := ensureSharedOrSlave(src, c.os.LookupMount); err != nil {
return err
}
options = append(options, "rslave")
if g.Config.Linux.RootfsPropagation != "rshared" &&
g.Config.Linux.RootfsPropagation != "rslave" {
g.SetLinuxRootPropagation("rslave") // nolint: errcheck
}
default:
logrus.Warnf("Unknown propagation mode for hostPath %q", mount.HostPath)
options = append(options, "rprivate")
}
// NOTE(random-liu): we don't change all mounts to `ro` when root filesystem
// is readonly. This is different from docker's behavior, but make more sense.
if mount.GetReadonly() {
options = append(options, "ro")
} else {
options = append(options, "rw")
}
if mount.GetSelinuxRelabel() {
if err := label.Relabel(src, mountLabel, true); err != nil && err != unix.ENOTSUP {
return errors.Wrapf(err, "relabel %q with %q failed", src, mountLabel)
}
}
g.AddMount(runtimespec.Mount{
Source: src,
Destination: dst,
Type: "bind",
Options: options,
})
}
return nil
}
func setOCIBindMountsPrivileged(g *generator) {
spec := g.Config
// clear readonly for /sys and cgroup
for i, m := range spec.Mounts {
if filepath.Clean(spec.Mounts[i].Destination) == "/sys" {
clearReadOnly(&spec.Mounts[i])
}
if m.Type == "cgroup" {
clearReadOnly(&spec.Mounts[i])
}
}
spec.Linux.ReadonlyPaths = nil
spec.Linux.MaskedPaths = nil
}
// setOCILinuxResourceCgroup set container cgroup resource limit.
func setOCILinuxResourceCgroup(g *generator, resources *runtime.LinuxContainerResources) {
if resources == nil {
return
}
g.SetLinuxResourcesCPUPeriod(uint64(resources.GetCpuPeriod()))
g.SetLinuxResourcesCPUQuota(resources.GetCpuQuota())
g.SetLinuxResourcesCPUShares(uint64(resources.GetCpuShares()))
g.SetLinuxResourcesMemoryLimit(resources.GetMemoryLimitInBytes())
g.SetLinuxResourcesCPUCpus(resources.GetCpusetCpus())
g.SetLinuxResourcesCPUMems(resources.GetCpusetMems())
}
// setOCILinuxResourceOOMScoreAdj set container OOMScoreAdj resource limit.
func setOCILinuxResourceOOMScoreAdj(g *generator, resources *runtime.LinuxContainerResources, restrictOOMScoreAdjFlag bool) error {
if resources == nil {
return nil
}
adj := int(resources.GetOomScoreAdj())
if restrictOOMScoreAdjFlag {
var err error
adj, err = restrictOOMScoreAdj(adj)
if err != nil {
return err
}
}
g.SetProcessOOMScoreAdj(adj)
return nil
}
// getOCICapabilitiesList returns a list of all available capabilities.
func getOCICapabilitiesList() []string {
var caps []string
for _, cap := range capability.List() {
if cap > validate.LastCap() {
continue
}
caps = append(caps, "CAP_"+strings.ToUpper(cap.String()))
}
return caps
}
// Adds capabilities to all sets relevant to root (bounding, permitted, effective, inheritable)
func addProcessRootCapability(g *generator, c string) error {
if err := g.AddProcessCapabilityBounding(c); err != nil {
return err
}
if err := g.AddProcessCapabilityPermitted(c); err != nil {
return err
}
if err := g.AddProcessCapabilityEffective(c); err != nil {
return err
}
if err := g.AddProcessCapabilityInheritable(c); err != nil {
return err
}
return nil
}
// Drops capabilities to all sets relevant to root (bounding, permitted, effective, inheritable)
func dropProcessRootCapability(g *generator, c string) error {
if err := g.DropProcessCapabilityBounding(c); err != nil {
return err
}
if err := g.DropProcessCapabilityPermitted(c); err != nil {
return err
}
if err := g.DropProcessCapabilityEffective(c); err != nil {
return err
}
if err := g.DropProcessCapabilityInheritable(c); err != nil {
return err
}
return nil
}
// setOCICapabilities adds/drops process capabilities.
func setOCICapabilities(g *generator, capabilities *runtime.Capability) error {
if capabilities == nil {
return nil
}
// Add/drop all capabilities if "all" is specified, so that
// following individual add/drop could still work. E.g.
// AddCapabilities: []string{"ALL"}, DropCapabilities: []string{"CHOWN"}
// will be all capabilities without `CAP_CHOWN`.
if util.InStringSlice(capabilities.GetAddCapabilities(), "ALL") {
for _, c := range getOCICapabilitiesList() {
if err := addProcessRootCapability(g, c); err != nil {
return err
}
}
}
if util.InStringSlice(capabilities.GetDropCapabilities(), "ALL") {
for _, c := range getOCICapabilitiesList() {
if err := dropProcessRootCapability(g, c); err != nil {
return err
}
}
}
for _, c := range capabilities.GetAddCapabilities() {
if strings.ToUpper(c) == "ALL" {
continue
}
// Capabilities in CRI doesn't have `CAP_` prefix, so add it.
if err := addProcessRootCapability(g, "CAP_"+strings.ToUpper(c)); err != nil {
return err
}
}
for _, c := range capabilities.GetDropCapabilities() {
if strings.ToUpper(c) == "ALL" {
continue
}
if err := dropProcessRootCapability(g, "CAP_"+strings.ToUpper(c)); err != nil {
return err
}
}
return nil
}
// setOCINamespaces sets namespaces.
func setOCINamespaces(g *generator, namespaces *runtime.NamespaceOption, sandboxPid uint32) {
g.AddOrReplaceLinuxNamespace(string(runtimespec.NetworkNamespace), getNetworkNamespace(sandboxPid)) // nolint: errcheck
g.AddOrReplaceLinuxNamespace(string(runtimespec.IPCNamespace), getIPCNamespace(sandboxPid)) // nolint: errcheck
g.AddOrReplaceLinuxNamespace(string(runtimespec.UTSNamespace), getUTSNamespace(sandboxPid)) // nolint: errcheck
// Do not share pid namespace if namespace mode is CONTAINER.
if namespaces.GetPid() != runtime.NamespaceMode_CONTAINER {
g.AddOrReplaceLinuxNamespace(string(runtimespec.PIDNamespace), getPIDNamespace(sandboxPid)) // nolint: errcheck
}
}
// defaultRuntimeSpec returns a default runtime spec used in cri-containerd.
func defaultRuntimeSpec(id string) (*runtimespec.Spec, error) {
// runtimeSpec returns a default runtime spec used in cri-containerd.
func runtimeSpec(id string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
// GenerateSpec needs namespace.
ctx := ctrdutil.NamespacedContext()
spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id})
spec, err := oci.GenerateSpec(ctx, nil, &containers.Container{ID: id}, opts...)
if err != nil {
return nil, err
}
// Remove `/run` mount
// TODO(random-liu): Mount tmpfs for /run and handle copy-up.
var mounts []runtimespec.Mount
for _, mount := range spec.Mounts {
if filepath.Clean(mount.Destination) == "/run" {
continue
}
mounts = append(mounts, mount)
}
spec.Mounts = mounts
// Make sure no default seccomp/apparmor is specified
if spec.Process != nil {
spec.Process.ApparmorProfile = ""
}
if spec.Linux != nil {
spec.Linux.Seccomp = nil
}
// Remove default rlimits (See issue #515)
spec.Process.Rlimits = nil
return spec, nil
}
@@ -1017,42 +587,6 @@ func generateApparmorSpecOpts(apparmorProf string, privileged, apparmorEnabled b
}
}
// Ensure mount point on which path is mounted, is shared.
func ensureShared(path string, lookupMount func(string) (mount.Info, error)) error {
mountInfo, err := lookupMount(path)
if err != nil {
return err
}
// Make sure source mount point is shared.
optsSplit := strings.Split(mountInfo.Optional, " ")
for _, opt := range optsSplit {
if strings.HasPrefix(opt, "shared:") {
return nil
}
}
return errors.Errorf("path %q is mounted on %q but it is not a shared mount", path, mountInfo.Mountpoint)
}
// Ensure mount point on which path is mounted, is either shared or slave.
func ensureSharedOrSlave(path string, lookupMount func(string) (mount.Info, error)) error {
mountInfo, err := lookupMount(path)
if err != nil {
return err
}
// Make sure source mount point is shared.
optsSplit := strings.Split(mountInfo.Optional, " ")
for _, opt := range optsSplit {
if strings.HasPrefix(opt, "shared:") {
return nil
} else if strings.HasPrefix(opt, "master:") {
return nil
}
}
return errors.Errorf("path %q is mounted on %q but it is not a shared or slave mount", path, mountInfo.Mountpoint)
}
// generateUserString generates valid user string based on OCI Image Spec v1.0.0.
// TODO(random-liu): Add group name support in CRI.
func generateUserString(username string, uid, gid *runtime.Int64Value) (string, error) {
@@ -1077,25 +611,3 @@ func generateUserString(username string, uid, gid *runtime.Int64Value) (string,
}
return userstr, nil
}
// mergeMounts merge CRI mounts with extra mounts. If a mount destination
// is mounted by both a CRI mount and an extra mount, the CRI mount will
// be kept.
func mergeMounts(criMounts, extraMounts []*runtime.Mount) []*runtime.Mount {
var mounts []*runtime.Mount
mounts = append(mounts, criMounts...)
// Copy all mounts from extra mounts, except for mounts overriden by CRI.
for _, e := range extraMounts {
found := false
for _, c := range criMounts {
if filepath.Clean(e.ContainerPath) == filepath.Clean(c.ContainerPath) {
found = true
break
}
}
if !found {
mounts = append(mounts, e)
}
}
return mounts
}

89
pkg/server/container_create_test.go
View File

@@ -29,13 +29,13 @@ import (
"github.com/containerd/containerd/oci"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"github.com/containerd/cri/pkg/annotations"
"github.com/containerd/cri/pkg/containerd/opts"
ostesting "github.com/containerd/cri/pkg/os/testing"
"github.com/containerd/cri/pkg/util"
)
@@ -118,7 +118,8 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
},
Annotations: map[string]string{"c": "d"},
Linux: &runtime.LinuxPodSandboxConfig{
CgroupParent: "/test/cgroup/parent",
CgroupParent: "/test/cgroup/parent",
SecurityContext: &runtime.LinuxSandboxSecurityContext{},
},
}
imageConfig := &imagespec.ImageConfig{
@@ -163,19 +164,19 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
t.Logf("Check namespaces")
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.NetworkNamespace,
Path: getNetworkNamespace(sandboxPid),
Path: opts.GetNetworkNamespace(sandboxPid),
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.IPCNamespace,
Path: getIPCNamespace(sandboxPid),
Path: opts.GetIPCNamespace(sandboxPid),
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.UTSNamespace,
Path: getUTSNamespace(sandboxPid),
Path: opts.GetUTSNamespace(sandboxPid),
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.PIDNamespace,
Path: getPIDNamespace(sandboxPid),
Path: opts.GetPIDNamespace(sandboxPid),
})
t.Logf("Check PodSandbox annotations")
@@ -203,8 +204,6 @@ func TestContainerCapabilities(t *testing.T) {
testID := "test-id"
testSandboxID := "sandbox-id"
testPid := uint32(1234)
config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
c := newTestCRIService()
for desc, test := range map[string]struct {
capability *runtime.Capability
includes []string
@@ -222,20 +221,20 @@ func TestContainerCapabilities(t *testing.T) {
capability: &runtime.Capability{
AddCapabilities: []string{"ALL"},
},
includes: getOCICapabilitiesList(),
includes: oci.GetAllCapabilities(),
},
"should be able to drop all capabilities": {
capability: &runtime.Capability{
DropCapabilities: []string{"ALL"},
},
excludes: getOCICapabilitiesList(),
excludes: oci.GetAllCapabilities(),
},
"should be able to drop capabilities with add all": {
capability: &runtime.Capability{
AddCapabilities: []string{"ALL"},
DropCapabilities: []string{"CHOWN"},
},
includes: util.SubtractStringSlice(getOCICapabilitiesList(), "CAP_CHOWN"),
includes: util.SubtractStringSlice(oci.GetAllCapabilities(), "CAP_CHOWN"),
excludes: []string{"CAP_CHOWN"},
},
"should be able to add capabilities with drop all": {
@@ -244,10 +243,13 @@ func TestContainerCapabilities(t *testing.T) {
DropCapabilities: []string{"ALL"},
},
includes: []string{"CAP_SYS_ADMIN"},
excludes: util.SubtractStringSlice(getOCICapabilitiesList(), "CAP_SYS_ADMIN"),
excludes: util.SubtractStringSlice(oci.GetAllCapabilities(), "CAP_SYS_ADMIN"),
},
} {
t.Logf("TestCase %q", desc)
config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
c := newTestCRIService()
config.Linux.SecurityContext.Capabilities = test.capability
spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil, []string{})
require.NoError(t, err)
@@ -522,20 +524,19 @@ func TestContainerSpecCommand(t *testing.T) {
} {
config, _, imageConfig, _ := getCreateContainerTestData()
og, err := generate.New("linux")
assert.NoError(t, err)
g := newCustomGenerator(og)
config.Command = test.criEntrypoint
config.Args = test.criArgs
imageConfig.Entrypoint = test.imageEntrypoint
imageConfig.Cmd = test.imageArgs
err = setOCIProcessArgs(&g, config, imageConfig)
var spec runtimespec.Spec
err := opts.WithProcessArgs(config, imageConfig)(nil, nil, nil, &spec)
if test.expectErr {
assert.Error(t, err)
continue
}
assert.NoError(t, err)
assert.Equal(t, test.expected, g.Config.Process.Args, desc)
assert.Equal(t, test.expected, spec.Process.Args, desc)
}
}
@@ -762,6 +763,11 @@ func TestGenerateContainerMounts(t *testing.T) {
}
func TestPrivilegedBindMount(t *testing.T) {
testPid := uint32(1234)
c := newTestCRIService()
testSandboxID := "sandbox-id"
config, sandboxConfig, imageConfig, _ := getCreateContainerTestData()
for desc, test := range map[string]struct {
privileged bool
expectedSysFSRO bool
@@ -778,15 +784,13 @@ func TestPrivilegedBindMount(t *testing.T) {
},
} {
t.Logf("TestCase %q", desc)
og, err := generate.New("linux")
config.Linux.SecurityContext.Privileged = test.privileged
sandboxConfig.Linux.SecurityContext.Privileged = test.privileged
spec, err := c.generateContainerSpec(t.Name(), testSandboxID, testPid, config, sandboxConfig, imageConfig, nil, nil)
assert.NoError(t, err)
g := newCustomGenerator(og)
c := newTestCRIService()
c.addOCIBindMounts(&g, nil, "")
if test.privileged {
setOCIBindMountsPrivileged(&g)
}
spec := g.Config
if test.expectedSysFSRO {
checkMount(t, spec.Mounts, "sysfs", "/sys", "sysfs", []string{"ro"}, []string{"rw"})
} else {
@@ -801,6 +805,7 @@ func TestPrivilegedBindMount(t *testing.T) {
}
func TestMountPropagation(t *testing.T) {
sharedLookupMountFn := func(string) (mount.Info, error) {
return mount.Info{
Mountpoint: "host-path",
@@ -888,17 +893,19 @@ func TestMountPropagation(t *testing.T) {
},
} {
t.Logf("TestCase %q", desc)
og, err := generate.New("linux")
assert.NoError(t, err)
g := newCustomGenerator(og)
c := newTestCRIService()
c.os.(*ostesting.FakeOS).LookupMountFn = test.fakeLookupMountFn
err = c.addOCIBindMounts(&g, []*runtime.Mount{test.criMount}, "")
config, _, _, _ := getCreateContainerTestData()
var spec runtimespec.Spec
spec.Linux = &runtimespec.Linux{}
err := opts.WithMounts(c.os, config, []*runtime.Mount{test.criMount}, "")(nil, nil, nil, &spec)
if test.expectErr {
require.Error(t, err)
} else {
require.NoError(t, err)
checkMount(t, g.Config.Mounts, test.criMount.HostPath, test.criMount.ContainerPath, "bind", test.optionsCheck, nil)
checkMount(t, spec.Mounts, test.criMount.HostPath, test.criMount.ContainerPath, "bind", test.optionsCheck, nil)
}
}
}
@@ -917,7 +924,7 @@ func TestPidNamespace(t *testing.T) {
pidNS: runtime.NamespaceMode_NODE,
expected: runtimespec.LinuxNamespace{
Type: runtimespec.PIDNamespace,
Path: getPIDNamespace(testPid),
Path: opts.GetPIDNamespace(testPid),
},
},
"container namespace mode": {
@@ -930,7 +937,7 @@ func TestPidNamespace(t *testing.T) {
pidNS: runtime.NamespaceMode_POD,
expected: runtimespec.LinuxNamespace{
Type: runtimespec.PIDNamespace,
Path: getPIDNamespace(testPid),
Path: opts.GetPIDNamespace(testPid),
},
},
} {
@@ -942,8 +949,14 @@ func TestPidNamespace(t *testing.T) {
}
}
func TestDefaultRuntimeSpec(t *testing.T) {
spec, err := defaultRuntimeSpec("test-id")
func TestNoDefaultRunMount(t *testing.T) {
testID := "test-id"
testPid := uint32(1234)
testSandboxID := "sandbox-id"
config, sandboxConfig, imageConfig, _ := getCreateContainerTestData()
c := newTestCRIService()
spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil, nil)
assert.NoError(t, err)
for _, mount := range spec.Mounts {
assert.NotEqual(t, "/run", mount.Destination)
@@ -1079,8 +1092,10 @@ func TestMaskedAndReadonlyPaths(t *testing.T) {
testPid := uint32(1234)
config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
c := newTestCRIService()
defaultSpec, err := defaultRuntimeSpec(testID)
defaultSpec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil, nil)
require.NoError(t, err)
for desc, test := range map[string]struct {
masked []string
readonly []string
@@ -1096,8 +1111,8 @@ func TestMaskedAndReadonlyPaths(t *testing.T) {
"should be able to specify empty paths": {
masked: []string{},
readonly: []string{},
expectedMasked: nil,
expectedReadonly: nil,
expectedMasked: []string{},
expectedReadonly: []string{},
privileged: false,
},
"should apply CRI specified paths": {

15
pkg/server/container_execsync.go
View File

@@ -24,6 +24,7 @@ import (
"github.com/containerd/containerd"
containerdio "github.com/containerd/containerd/cio"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/oci"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
@@ -99,14 +100,16 @@ func (c *criService) execInContainer(ctx context.Context, id string, opts execOp
if err != nil {
return nil, errors.Wrap(err, "failed to load task")
}
if opts.tty {
g := newSpecGenerator(spec)
g.AddProcessEnv("TERM", "xterm")
spec = g.Config
}
pspec := spec.Process
pspec.Args = opts.cmd
pspec.Terminal = opts.tty
if opts.tty {
if err := oci.WithEnv([]string{"TERM=xterm"})(nil, nil, nil, spec); err != nil {
return nil, errors.Wrap(err, "add TERM env var to spec")
}
}
pspec.Args = opts.cmd
if opts.stdout == nil {
opts.stdout = cio.NewDiscardLogger()

27
pkg/server/container_update_resources.go
View File

@@ -29,6 +29,7 @@ import (
"golang.org/x/net/context"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"github.com/containerd/cri/pkg/containerd/opts"
ctrdutil "github.com/containerd/cri/pkg/containerd/util"
containerstore "github.com/containerd/cri/pkg/store/container"
"github.com/containerd/cri/pkg/util"
@@ -135,27 +136,11 @@ func updateOCILinuxResource(spec *runtimespec.Spec, new *runtime.LinuxContainerR
if err := util.DeepCopy(&cloned, spec); err != nil {
return nil, errors.Wrap(err, "failed to deep copy")
}
g := newSpecGenerator(&cloned)
if new.GetCpuPeriod() != 0 {
g.SetLinuxResourcesCPUPeriod(uint64(new.GetCpuPeriod()))
if cloned.Linux == nil {
cloned.Linux = &runtimespec.Linux{}
}
if new.GetCpuQuota() != 0 {
g.SetLinuxResourcesCPUQuota(new.GetCpuQuota())
if err := opts.WithResources(new)(nil, nil, nil, &cloned); err != nil {
return nil, errors.Wrap(err, "unable to set linux container resources")
}
if new.GetCpuShares() != 0 {
g.SetLinuxResourcesCPUShares(uint64(new.GetCpuShares()))
}
if new.GetMemoryLimitInBytes() != 0 {
g.SetLinuxResourcesMemoryLimit(new.GetMemoryLimitInBytes())
}
// OOMScore is not updatable.
if new.GetCpusetCpus() != "" {
g.SetLinuxResourcesCPUCpus(new.GetCpusetCpus())
}
if new.GetCpusetMems() != "" {
g.SetLinuxResourcesCPUMems(new.GetCpusetMems())
}
return g.Config, nil
return &cloned, nil
}

133
pkg/server/helpers.go
View File

@@ -18,8 +18,6 @@ package server
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
@@ -36,8 +34,6 @@ import (
"github.com/containerd/typeurl"
"github.com/docker/distribution/reference"
imagedigest "github.com/opencontainers/go-digest"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"github.com/opencontainers/selinux/go-selinux"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
@@ -69,8 +65,6 @@ const (
const (
// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
defaultSandboxOOMAdj = -998
// defaultSandboxCPUshares is default cpu shares for sandbox container.
defaultSandboxCPUshares = 2
// defaultShmSize is the default size of the sandbox shm.
defaultShmSize = int64(1024 * 1024 * 64)
// relativeRootfsPath is the rootfs path relative to bundle path.
@@ -86,14 +80,6 @@ const (
maxDNSSearches = 6
// Delimiter used to construct container/sandbox names.
nameDelimiter = "_"
// netNSFormat is the format of network namespace of a process.
netNSFormat = "/proc/%v/ns/net"
// ipcNSFormat is the format of ipc namespace of a process.
ipcNSFormat = "/proc/%v/ns/ipc"
// utsNSFormat is the format of uts namespace of a process.
utsNSFormat = "/proc/%v/ns/uts"
// pidNSFormat is the format of pid namespace of a process.
pidNSFormat = "/proc/%v/ns/pid"
// devShm is the default path of /dev/shm.
devShm = "/dev/shm"
// etcHosts is the default path of /etc/hosts file.
@@ -220,26 +206,6 @@ func (c *criService) getSandboxDevShm(id string) string {
return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
}
// getNetworkNamespace returns the network namespace of a process.
func getNetworkNamespace(pid uint32) string {
return fmt.Sprintf(netNSFormat, pid)
}
// getIPCNamespace returns the ipc namespace of a process.
func getIPCNamespace(pid uint32) string {
return fmt.Sprintf(ipcNSFormat, pid)
}
// getUTSNamespace returns the uts namespace of a process.
func getUTSNamespace(pid uint32) string {
return fmt.Sprintf(utsNSFormat, pid)
}
// getPIDNamespace returns the pid namespace of a process.
func getPIDNamespace(pid uint32) string {
return fmt.Sprintf(pidNSFormat, pid)
}
// criContainerStateToString formats CRI container state to string.
func criContainerStateToString(state runtime.ContainerState) string {
return runtime.ContainerState_name[int32(state)]
@@ -397,54 +363,6 @@ func buildLabels(configLabels map[string]string, containerType string) map[strin
return labels
}
// newSpecGenerator creates a new spec generator for the runtime spec.
func newSpecGenerator(spec *runtimespec.Spec) generator {
g := generate.NewFromSpec(spec)
g.HostSpecific = true
return newCustomGenerator(g)
}
// generator is a custom generator with some functions overridden
// used by the cri plugin.
// TODO(random-liu): Upstream this fix.
type generator struct {
generate.Generator
envCache map[string]int
}
func newCustomGenerator(g generate.Generator) generator {
cg := generator{
Generator: g,
envCache: make(map[string]int),
}
if g.Config != nil && g.Config.Process != nil {
for i, env := range g.Config.Process.Env {
kv := strings.SplitN(env, "=", 2)
cg.envCache[kv[0]] = i
}
}
return cg
}
// AddProcessEnv overrides the original AddProcessEnv. It uses
// a map to cache and override envs.
func (g *generator) AddProcessEnv(key, value string) {
if len(g.envCache) == 0 {
// Call AddProccessEnv once to initialize the spec.
g.Generator.AddProcessEnv(key, value)
g.envCache[key] = 0
return
}
spec := g.Config
env := fmt.Sprintf("%s=%s", key, value)
if idx, ok := g.envCache[key]; !ok {
spec.Process.Env = append(spec.Process.Env, env)
g.envCache[key] = len(spec.Process.Env) - 1
} else {
spec.Process.Env[idx] = env
}
}
func getPodCNILabels(id string, config *runtime.PodSandboxConfig) map[string]string {
return map[string]string{
"K8S_POD_NAMESPACE": config.GetMetadata().GetNamespace(),
@@ -464,33 +382,6 @@ func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
}
}
// mounts defines how to sort runtime.Mount.
// This is the same with the Docker implementation:
// https://github.com/moby/moby/blob/17.05.x/daemon/volumes.go#L26
type orderedMounts []*runtime.Mount
// Len returns the number of mounts. Used in sorting.
func (m orderedMounts) Len() int {
return len(m)
}
// Less returns true if the number of parts (a/b/c would be 3 parts) in the
// mount indexed by parameter 1 is less than that of the mount indexed by
// parameter 2. Used in sorting.
func (m orderedMounts) Less(i, j int) bool {
return m.parts(i) < m.parts(j)
}
// Swap swaps two items in an array of mounts. Used in sorting
func (m orderedMounts) Swap(i, j int) {
m[i], m[j] = m[j], m[i]
}
// parts returns the number of parts in the destination of a mount. Used in sorting.
func (m orderedMounts) parts(i int) int {
return strings.Count(filepath.Clean(m[i].ContainerPath), string(os.PathSeparator))
}
// parseImageReferences parses a list of arbitrary image references and returns
// the repotags and repodigests
func parseImageReferences(refs []string) ([]string, []string) {
@@ -553,30 +444,6 @@ func getRuntimeOptions(c containers.Container) (interface{}, error) {
return opts, nil
}
func getCurrentOOMScoreAdj() (int, error) {
b, err := ioutil.ReadFile("/proc/self/oom_score_adj")
if err != nil {
return 0, errors.Wrap(err, "could not get the daemon oom_score_adj")
}
s := strings.TrimSpace(string(b))
i, err := strconv.Atoi(s)
if err != nil {
return 0, errors.Wrap(err, "could not get the daemon oom_score_adj")
}
return i, nil
}
func restrictOOMScoreAdj(preferredOOMScoreAdj int) (int, error) {
currentOOMScoreAdj, err := getCurrentOOMScoreAdj()
if err != nil {
return preferredOOMScoreAdj, err
}
if preferredOOMScoreAdj < currentOOMScoreAdj {
return currentOOMScoreAdj, nil
}
return preferredOOMScoreAdj, nil
}
const (
// unknownExitCode is the exit code when exit reason is unknown.
unknownExitCode = 255

70
pkg/server/helpers_test.go
View File

@@ -17,10 +17,10 @@ limitations under the License.
package server
import (
"sort"
"testing"
"github.com/BurntSushi/toml"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/runtime/linux/runctypes"
runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
"github.com/docker/distribution/reference"
@@ -28,7 +28,6 @@ import (
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
criconfig "github.com/containerd/cri/pkg/config"
"github.com/containerd/cri/pkg/store"
@@ -153,27 +152,6 @@ func TestBuildLabels(t *testing.T) {
assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
}
func TestOrderedMounts(t *testing.T) {
mounts := []*runtime.Mount{
{ContainerPath: "/a/b/c"},
{ContainerPath: "/a/b"},
{ContainerPath: "/a/b/c/d"},
{ContainerPath: "/a"},
{ContainerPath: "/b"},
{ContainerPath: "/b/c"},
}
expected := []*runtime.Mount{
{ContainerPath: "/a"},
{ContainerPath: "/b"},
{ContainerPath: "/a/b"},
{ContainerPath: "/b/c"},
{ContainerPath: "/a/b/c"},
{ContainerPath: "/a/b/c/d"},
}
sort.Stable(orderedMounts(mounts))
assert.Equal(t, expected, mounts)
}
func TestParseImageReferences(t *testing.T) {
refs := []string{
"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
@@ -306,33 +284,12 @@ systemd_cgroup = true
}
}
func TestRestrictOOMScoreAdj(t *testing.T) {
current, err := getCurrentOOMScoreAdj()
require.NoError(t, err)
got, err := restrictOOMScoreAdj(current - 1)
require.NoError(t, err)
assert.Equal(t, got, current)
got, err = restrictOOMScoreAdj(current)
require.NoError(t, err)
assert.Equal(t, got, current)
got, err = restrictOOMScoreAdj(current + 1)
require.NoError(t, err)
assert.Equal(t, got, current+1)
}
func TestCustomGenerator(t *testing.T) {
func TestEnvDeduplication(t *testing.T) {
for desc, test := range map[string]struct {
existing []string
kv [][2]string
expected []string
expectNil bool
existing []string
kv [][2]string
expected []string
}{
"empty": {
expectNil: true,
},
"single env": {
kv: [][2]string{
{"a", "b"},
@@ -387,23 +344,16 @@ func TestCustomGenerator(t *testing.T) {
},
} {
t.Logf("TestCase %q", desc)
var spec *runtimespec.Spec
var spec runtimespec.Spec
if len(test.existing) > 0 {
spec = &runtimespec.Spec{
Process: &runtimespec.Process{
Env: test.existing,
},
spec.Process = &runtimespec.Process{
Env: test.existing,
}
}
g := newSpecGenerator(spec)
for _, kv := range test.kv {
g.AddProcessEnv(kv[0], kv[1])
}
if test.expectNil {
assert.Nil(t, g.Config)
} else {
assert.Equal(t, test.expected, g.Config.Process.Env)
oci.WithEnv([]string{kv[0] + "=" + kv[1]})(nil, nil, nil, &spec)
}
assert.Equal(t, test.expected, spec.Process.Env)
}
}

104
pkg/server/sandbox_run.go
View File

@@ -343,67 +343,61 @@ func (c *criService) generateSandboxContainerSpec(id string, config *runtime.Pod
imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
// Creates a spec Generator with the default spec.
// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
spec, err := defaultRuntimeSpec(id)
if err != nil {
return nil, err
specOpts := []oci.SpecOpts{
customopts.WithoutRunMount,
customopts.WithoutDefaultSecuritySettings,
customopts.WithRelativeRoot(relativeRootfsPath),
oci.WithEnv(imageConfig.Env),
oci.WithRootFSReadonly(),
oci.WithHostname(config.GetHostname()),
}
g := newSpecGenerator(spec)
// Apply default config from image config.
if err := addImageEnvs(&g, imageConfig.Env); err != nil {
return nil, err
}
if imageConfig.WorkingDir != "" {
g.SetProcessCwd(imageConfig.WorkingDir)
specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
}
if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
// Pause image must have entrypoint or cmd.
return nil, errors.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
}
// Set process commands.
g.SetProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...))
// Set relative root path.
g.SetRootPath(relativeRootfsPath)
// Make root of sandbox container read-only.
g.SetRootReadonly(true)
// Set hostname.
g.SetHostname(config.GetHostname())
specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
// TODO(random-liu): [P2] Consider whether to add labels and annotations to the container.
// Set cgroups parent.
if c.config.DisableCgroup {
g.SetLinuxCgroupsPath("")
specOpts = append(specOpts, customopts.WithDisabledCgroups)
} else {
if config.GetLinux().GetCgroupParent() != "" {
cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id,
c.config.SystemdCgroup)
g.SetLinuxCgroupsPath(cgroupsPath)
specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
}
}
// When cgroup parent is not set, containerd-shim will create container in a child cgroup
// of the cgroup itself is in.
// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
// Set namespace options.
securityContext := config.GetLinux().GetSecurityContext()
nsOptions := securityContext.GetNamespaceOptions()
var (
securityContext = config.GetLinux().GetSecurityContext()
nsOptions = securityContext.GetNamespaceOptions()
)
if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE {
g.RemoveLinuxNamespace(string(runtimespec.NetworkNamespace)) // nolint: errcheck
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace))
} else {
//TODO(Abhi): May be move this to containerd spec opts (WithLinuxSpaceOption)
g.AddOrReplaceLinuxNamespace(string(runtimespec.NetworkNamespace), nsPath) // nolint: errcheck
specOpts = append(specOpts, oci.WithLinuxNamespace(
runtimespec.LinuxNamespace{
Type: runtimespec.NetworkNamespace,
Path: nsPath,
}))
}
if nsOptions.GetPid() == runtime.NamespaceMode_NODE {
g.RemoveLinuxNamespace(string(runtimespec.PIDNamespace)) // nolint: errcheck
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace))
}
if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
g.RemoveLinuxNamespace(string(runtimespec.IPCNamespace)) // nolint: errcheck
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace))
}
// It's fine to generate the spec before the sandbox /dev/shm
@@ -412,56 +406,50 @@ func (c *criService) generateSandboxContainerSpec(id string, config *runtime.Pod
if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
sandboxDevShm = devShm
}
g.AddMount(runtimespec.Mount{
Source: sandboxDevShm,
Destination: devShm,
Type: "bind",
Options: []string{"rbind", "ro"},
})
specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{
{
Source: sandboxDevShm,
Destination: devShm,
Type: "bind",
Options: []string{"rbind", "ro"},
},
}))
selinuxOpt := securityContext.GetSelinuxOptions()
processLabel, mountLabel, err := initSelinuxOpts(selinuxOpt)
if err != nil {
return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
}
g.SetProcessSelinuxLabel(processLabel)
g.SetLinuxMountLabel(mountLabel)
supplementalGroups := securityContext.GetSupplementalGroups()
for _, group := range supplementalGroups {
g.AddProcessAdditionalGid(uint32(group))
}
specOpts = append(specOpts,
customopts.WithSelinuxLabels(processLabel, mountLabel),
customopts.WithSupplementalGroups(supplementalGroups),
)
// Add sysctls
sysctls := config.GetLinux().GetSysctls()
for key, value := range sysctls {
g.AddLinuxSysctl(key, value)
}
specOpts = append(specOpts, customopts.WithSysctls(sysctls))
// Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile
if !c.config.DisableCgroup {
g.SetLinuxResourcesCPUShares(uint64(defaultSandboxCPUshares))
specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
}
adj := int(defaultSandboxOOMAdj)
if c.config.RestrictOOMScoreAdj {
adj, err = restrictOOMScoreAdj(adj)
if err != nil {
return nil, err
}
}
g.SetProcessOOMScoreAdj(adj)
specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
runtimePodAnnotations) {
g.AddAnnotation(pKey, pValue)
specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
}
g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox)
g.AddAnnotation(annotations.SandboxID, id)
g.AddAnnotation(annotations.SandboxLogDir, config.GetLogDirectory())
specOpts = append(specOpts,
customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
customopts.WithAnnotation(annotations.SandboxID, id),
customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
)
return g.Config, nil
return runtimeSpec(id, specOpts...)
}
// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,

10
pkg/server/sandbox_run_test.go
View File

@@ -32,6 +32,7 @@ import (
"github.com/containerd/cri/pkg/annotations"
criconfig "github.com/containerd/cri/pkg/config"
"github.com/containerd/cri/pkg/containerd/opts"
ostesting "github.com/containerd/cri/pkg/os/testing"
sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
)
@@ -66,7 +67,7 @@ func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConf
assert.Contains(t, spec.Process.Env, "a=b", "c=d")
assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args)
assert.Equal(t, "/workspace", spec.Process.Cwd)
assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, defaultSandboxCPUshares)
assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, opts.DefaultSandboxCPUshares)
assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj)
t.Logf("Check PodSandbox annotations")
@@ -139,13 +140,6 @@ func TestGenerateSandboxContainerSpec(t *testing.T) {
},
expectErr: true,
},
"should return error when env is invalid ": {
// Also covers addImageEnvs.
imageConfigChange: func(c *imagespec.ImageConfig) {
c.Env = []string{"a"}
},
expectErr: true,
},
"should set supplemental groups correctly": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{