From 20419feaac1963de082d0bfad9eb76c93c782df0 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 6 Oct 2021 14:50:35 -0700 Subject: [PATCH] cri, sandbox: pass sandbox resource details if available, applicable CRI API has been updated to include a an optional `resources` field in the LinuxPodSandboxConfig field, as part of the RunPodSandbox request. Having sandbox level resource details at sandbox creation time will have large benefits for sandboxed runtimes. In the case of Kata Containers, for example, this'll allow for better support of SW/HW architectures which don't allow for CPU/memory hotplug, and it'll allow for better queue sizing for virtio devices associated with the sandbox (in the VM case). If this sandbox resource information is provided as part of the run sandbox request, let's introduce a pattern where we will update the pause container's runtiem spec to include this information in the annotations field. Signed-off-by: Eric Ernst --- pkg/cri/annotations/annotations.go | 10 ++++ pkg/cri/server/sandbox_run_linux.go | 10 ++++ pkg/cri/server/sandbox_run_linux_test.go | 61 ++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/pkg/cri/annotations/annotations.go b/pkg/cri/annotations/annotations.go index 1cd8f5ccc..181eb05c7 100644 --- a/pkg/cri/annotations/annotations.go +++ b/pkg/cri/annotations/annotations.go @@ -32,6 +32,16 @@ const ( // SandboxID is the sandbox ID annotation SandboxID = "io.kubernetes.cri.sandbox-id" + // SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the + // sum of container CPU resources, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig + SandboxCPUPeriod = "io.kubernetes.cri.sandbox-cpu-period" + SandboxCPUQuota = "io.kubernetes.cri.sandbox-cpu-quota" + SandboxCPUShares = "io.kubernetes.cri.sandbox-cpu-shares" + + // SandboxMemory is the initial amount of memory associated with this sandbox. This is calculated as the sum + // of container memory, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig. + SandboxMem = "io.kubernetes.cri.sandbox-memory" + // SandboxLogDir is the pod log directory annotation. // If the sandbox needs to generate any log, it will put it into this directory. // Kubelet will be responsible for: diff --git a/pkg/cri/server/sandbox_run_linux.go b/pkg/cri/server/sandbox_run_linux.go index 82c197310..4a82f1a9a 100644 --- a/pkg/cri/server/sandbox_run_linux.go +++ b/pkg/cri/server/sandbox_run_linux.go @@ -19,6 +19,7 @@ package server import ( "fmt" "os" + "strconv" "strings" "github.com/containerd/containerd" @@ -155,6 +156,15 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC if !c.config.DisableCgroup { specOpts = append(specOpts, customopts.WithDefaultSandboxShares) } + + if res := config.GetLinux().GetResources(); res != nil { + specOpts = append(specOpts, + customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)), + customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)), + customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)), + customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10))) + } + specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj)) for pKey, pValue := range getPassthroughAnnotations(config.Annotations, diff --git a/pkg/cri/server/sandbox_run_linux_test.go b/pkg/cri/server/sandbox_run_linux_test.go index b203984e2..22509b379 100644 --- a/pkg/cri/server/sandbox_run_linux_test.go +++ b/pkg/cri/server/sandbox_run_linux_test.go @@ -19,6 +19,7 @@ package server import ( "os" "path/filepath" + "strconv" "testing" imagespec "github.com/opencontainers/image-spec/specs-go/v1" @@ -27,6 +28,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + v1 "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/containerd/containerd/pkg/cri/annotations" "github.com/containerd/containerd/pkg/cri/opts" @@ -173,6 +175,65 @@ func TestLinuxSandboxContainerSpec(t *testing.T) { assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "1 1000") }, }, + "sandbox sizing annotations should be set if LinuxContainerResources were provided": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Linux.Resources = &v1.LinuxContainerResources{ + CpuPeriod: 100, + CpuQuota: 200, + CpuShares: 5000, + MemoryLimitInBytes: 1024, + } + }, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + value, ok := spec.Annotations[annotations.SandboxCPUPeriod] + assert.True(t, ok) + assert.EqualValues(t, strconv.FormatInt(100, 10), value) + assert.EqualValues(t, "100", value) + + value, ok = spec.Annotations[annotations.SandboxCPUQuota] + assert.True(t, ok) + assert.EqualValues(t, "200", value) + + value, ok = spec.Annotations[annotations.SandboxCPUShares] + assert.True(t, ok) + assert.EqualValues(t, "5000", value) + + value, ok = spec.Annotations[annotations.SandboxMem] + assert.True(t, ok) + assert.EqualValues(t, "1024", value) + }, + }, + "sandbox sizing annotations should not be set if LinuxContainerResources were not provided": { + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + _, ok := spec.Annotations[annotations.SandboxCPUPeriod] + assert.False(t, ok) + _, ok = spec.Annotations[annotations.SandboxCPUQuota] + assert.False(t, ok) + _, ok = spec.Annotations[annotations.SandboxCPUShares] + assert.False(t, ok) + _, ok = spec.Annotations[annotations.SandboxMem] + assert.False(t, ok) + }, + }, + "sandbox sizing annotations are zero if the resources are set to 0": { + configChange: func(c *runtime.PodSandboxConfig) { + c.Linux.Resources = &v1.LinuxContainerResources{} + }, + specCheck: func(t *testing.T, spec *runtimespec.Spec) { + value, ok := spec.Annotations[annotations.SandboxCPUPeriod] + assert.True(t, ok) + assert.EqualValues(t, "0", value) + value, ok = spec.Annotations[annotations.SandboxCPUQuota] + assert.True(t, ok) + assert.EqualValues(t, "0", value) + value, ok = spec.Annotations[annotations.SandboxCPUShares] + assert.True(t, ok) + assert.EqualValues(t, "0", value) + value, ok = spec.Annotations[annotations.SandboxMem] + assert.True(t, ok) + assert.EqualValues(t, "0", value) + }, + }, } { t.Logf("TestCase %q", desc) c := newTestCRIService()