Annotations: Provide container metadata for VM based runtimes

For hypervisor-based container runtimes (like Kata Containers, Clear Containers
or runv) a pod will be created in a VM and then create containers within the VM.

When a runtime is requested for container commands like create and start, both
the instal "pause" container and next containers need to be added to the pod
namespace (same VM).

A runtime does not know if it needs to create/start a VM or if it needs to add a
container to an already running VM pod.

This patch adds a way to provide this information through container annotations.
When starting a container or a sandbox, 2 annotations are added:

- type (Container or Sandbox)
- sandbox name

This allow to a VM based runtime to decide if they need to create a pod VM or
container within the VM pod.

Signed-off-by: Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>

pkg/server/container_create.go

@@ -45,6 +45,7 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
+	"github.com/containerd/cri-containerd/pkg/annotations"
 	customopts "github.com/containerd/cri-containerd/pkg/containerd/opts"
 	cio "github.com/containerd/cri-containerd/pkg/server/io"
 	containerstore "github.com/containerd/cri-containerd/pkg/store/container"
@@ -143,10 +144,11 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 	// Generate container runtime spec.
 	mounts := c.generateContainerMounts(getSandboxRootDir(c.config.RootDir, sandboxID), config)
 
-	spec, err := c.generateContainerSpec(id, sandboxPid, config, sandboxConfig, &image.ImageSpec.Config, append(mounts, volumeMounts...))
+	spec, err := c.generateContainerSpec(id, sandboxID, sandboxPid, config, sandboxConfig, &image.ImageSpec.Config, append(mounts, volumeMounts...))
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate container %q spec: %v", id, err)
 	}
+
 	glog.V(4).Infof("Container %q spec: %#+v", id, spew.NewFormatter(spec))
 
 	// Set snapshotter before any other options.
@@ -271,7 +273,7 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 	return &runtime.CreateContainerResponse{ContainerId: id}, nil
 }
 
-func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint32, config *runtime.ContainerConfig,
+func (c *criContainerdService) generateContainerSpec(id string, sandboxID string, sandboxPid uint32, config *runtime.ContainerConfig,
 	sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig, extraMounts []*runtime.Mount) (*runtimespec.Spec, error) {
 	// Creates a spec Generator with the default spec.
 	spec, err := defaultRuntimeSpec(id)
@@ -365,6 +367,9 @@ func (c *criContainerdService) generateContainerSpec(id string, sandboxPid uint3
 		g.AddProcessAdditionalGid(uint32(group))
 	}
 
+	g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer)
+	g.AddAnnotation(annotations.SandboxID, sandboxID)
+
 	return g.Spec(), nil
 }
 

pkg/server/container_create_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
+	"github.com/containerd/cri-containerd/pkg/annotations"
 	ostesting "github.com/containerd/cri-containerd/pkg/os/testing"
 	"github.com/containerd/cri-containerd/pkg/util"
 )
@@ -56,7 +57,7 @@ func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string,
 }
 
 func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig,
-	*imagespec.ImageConfig, func(*testing.T, string, uint32, *runtimespec.Spec)) {
+	*imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) {
 	config := &runtime.ContainerConfig{
 		Metadata: &runtime.ContainerMetadata{
 			Name:    "test-name",
@@ -122,7 +123,7 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
 		Cmd:        []string{"cmd"},
 		WorkingDir: "/workspace",
 	}
-	specCheck := func(t *testing.T, id string, sandboxPid uint32, spec *runtimespec.Spec) {
+	specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) {
 		assert.Equal(t, relativeRootfsPath, spec.Root.Path)
 		assert.Equal(t, []string{"test", "command", "test", "args"}, spec.Process.Args)
 		assert.Equal(t, "test-cwd", spec.Process.Cwd)
@@ -168,6 +169,13 @@ func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandbox
 			Type: runtimespec.UTSNamespace,
 			Path: getUTSNamespace(sandboxPid),
 		})
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], sandboxID)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeContainer)
 	}
 	return config, sandboxConfig, imageConfig, specCheck
 }
@@ -177,13 +185,15 @@ func TestGeneralContainerSpec(t *testing.T) {
 	testPid := uint32(1234)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
-	spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+	testSandboxID := "SandboxID"
+	spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 	require.NoError(t, err)
-	specCheck(t, testID, testPid, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
 }
 
 func TestContainerCapabilities(t *testing.T) {
 	testID := "test-id"
+	testSandboxID := "SandboxID"
 	testPid := uint32(1234)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
@@ -231,9 +241,9 @@ func TestContainerCapabilities(t *testing.T) {
 	} {
 		t.Logf("TestCase %q", desc)
 		config.Linux.SecurityContext.Capabilities = test.capability
-		spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+		spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 		require.NoError(t, err)
-		specCheck(t, testID, testPid, spec)
+		specCheck(t, testID, testSandboxID, testPid, spec)
 		t.Log(spec.Process.Capabilities.Bounding)
 		for _, include := range test.includes {
 			assert.Contains(t, spec.Process.Capabilities.Bounding, include)
@@ -252,14 +262,15 @@ func TestContainerCapabilities(t *testing.T) {
 
 func TestContainerSpecTty(t *testing.T) {
 	testID := "test-id"
+	testSandboxID := "SandboxID"
 	testPid := uint32(1234)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
 	for _, tty := range []bool{true, false} {
 		config.Tty = tty
-		spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+		spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 		require.NoError(t, err)
-		specCheck(t, testID, testPid, spec)
+		specCheck(t, testID, testSandboxID, testPid, spec)
 		assert.Equal(t, tty, spec.Process.Terminal)
 		if tty {
 			assert.Contains(t, spec.Process.Env, "TERM=xterm")
@@ -271,20 +282,22 @@ func TestContainerSpecTty(t *testing.T) {
 
 func TestContainerSpecReadonlyRootfs(t *testing.T) {
 	testID := "test-id"
+	testSandboxID := "SandboxID"
 	testPid := uint32(1234)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
 	for _, readonly := range []bool{true, false} {
 		config.Linux.SecurityContext.ReadonlyRootfs = readonly
-		spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+		spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 		require.NoError(t, err)
-		specCheck(t, testID, testPid, spec)
+		specCheck(t, testID, testSandboxID, testPid, spec)
 		assert.Equal(t, readonly, spec.Root.Readonly)
 	}
 }
 
 func TestContainerSpecWithExtraMounts(t *testing.T) {
 	testID := "test-id"
+	testSandboxID := "SandboxID"
 	testPid := uint32(1234)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
@@ -299,9 +312,9 @@ func TestContainerSpecWithExtraMounts(t *testing.T) {
 		HostPath:      "test-host-path-extra",
 		Readonly:      true,
 	}
-	spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, []*runtime.Mount{extraMount})
+	spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, []*runtime.Mount{extraMount})
 	require.NoError(t, err)
-	specCheck(t, testID, testPid, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
 	var mounts []runtimespec.Mount
 	for _, m := range spec.Mounts {
 		if m.Destination == "test-container-path" {
@@ -688,22 +701,23 @@ func TestMountPropagation(t *testing.T) {
 func TestPidNamespace(t *testing.T) {
 	testID := "test-id"
 	testPid := uint32(1234)
+	testSandboxID := "SandboxID"
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
 	c := newTestCRIContainerdService()
 	t.Logf("should not set pid namespace when host pid is true")
 	config.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{HostPid: true}
-	spec, err := c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+	spec, err := c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 	require.NoError(t, err)
-	specCheck(t, testID, testPid, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
 	for _, ns := range spec.Linux.Namespaces {
 		assert.NotEqual(t, ns.Type, runtimespec.PIDNamespace)
 	}
 
 	t.Logf("should set pid namespace when host pid is false")
 	config.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{HostPid: false}
-	spec, err = c.generateContainerSpec(testID, testPid, config, sandboxConfig, imageConfig, nil)
+	spec, err = c.generateContainerSpec(testID, testSandboxID, testPid, config, sandboxConfig, imageConfig, nil)
 	require.NoError(t, err)
-	specCheck(t, testID, testPid, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
 	assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
 		Type: runtimespec.PIDNamespace,
 	})

pkg/server/sandbox_run.go

@@ -34,6 +34,7 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
+	"github.com/containerd/cri-containerd/pkg/annotations"
 	customopts "github.com/containerd/cri-containerd/pkg/containerd/opts"
 	sandboxstore "github.com/containerd/cri-containerd/pkg/store/sandbox"
 	"github.com/containerd/cri-containerd/pkg/util"
@@ -331,6 +332,9 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
 	g.SetLinuxResourcesCPUShares(uint64(defaultSandboxCPUshares))
 	g.SetProcessOOMScoreAdj(int(defaultSandboxOOMAdj))
 
+	g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox)
+	g.AddAnnotation(annotations.SandboxID, id)
+
 	return g.Spec(), nil
 }
 

pkg/server/sandbox_run_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/containerd/cri-containerd/pkg/annotations"
 	"github.com/containerd/typeurl"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -64,6 +65,13 @@ func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConf
 		assert.Equal(t, "/workspace", spec.Process.Cwd)
 		assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, defaultSandboxCPUshares)
 		assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj)
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox)
 	}
 	return config, imageConfig, specCheck
 }