Merge pull request #260 from yanxuean/use-containerd-extension

Switch to containerd extension

pkg/server/container_create.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/contrib/apparmor"
+	"github.com/containerd/containerd/typeurl"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/golang/glog"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -55,6 +56,11 @@ const (
 	appArmorEnabled = true // TODO (mikebrow): make these apparmor defaults configurable
 )
 
+func init() {
+	typeurl.Register(&containerstore.Metadata{},
+		"github.com/kubernetes-incubator/cri-containerd/pkg/store/container", "Metadata")
+}
+
 // CreateContainer creates a new container in the given PodSandbox.
 func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) {
 	config := r.GetConfig()
@@ -169,14 +175,6 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 		}
 	}()
 
-	metaBytes, err := meta.Encode()
-	if err != nil {
-		return nil, fmt.Errorf("failed to convert sandbox metadata: %+v, %v", meta, err)
-	}
-	labels := map[string]string{
-		containerMetadataLabel: string(metaBytes),
-	}
-
 	var specOpts []containerd.SpecOpts
 	// Set container username. This could only be done by containerd, because it needs
 	// access to the container rootfs. Pass user name to containerd, and let it overwrite
@@ -207,7 +205,8 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 	opts = append(opts,
 		containerd.WithSpec(spec, specOpts...),
 		containerd.WithRuntime(defaultRuntime, nil),
-		containerd.WithContainerLabels(labels))
+		containerd.WithContainerLabels(map[string]string{containerKindLabel: containerKindContainer}),
+		containerd.WithContainerExtension(containerMetadataExtension, &meta))
 	var cntr containerd.Container
 	if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil {
 		return nil, fmt.Errorf("failed to create containerd container: %v", err)

pkg/server/helpers.go

@@ -95,10 +95,18 @@ const (
 )
 
 const (
-	// sandboxMetadataLabel is label name that identify metadata of sandbox in CreateContainerRequest
+	// criContainerdPrefix is common prefix for cri-containerd
-	sandboxMetadataLabel = "io.cri-containerd.sandbox.metadata"
+	criContainerdPrefix = "io.cri-containerd"
-	// sandboxMetadataLabel is label name that identify metadata of container in CreateContainerRequest
+	// containerKindLabel is a label key indicating container is sandbox container or application container
-	containerMetadataLabel = "io.cri-containerd.container.metadata"
+	containerKindLabel = criContainerdPrefix + ".kind"
+	// containerKindSandbox is a label value indicating container is sandbox container
+	containerKindSandbox = "sandbox"
+	// containerKindContainer is a label value indicating container is application container
+	containerKindContainer = "container"
+	// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
+	sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
+	// containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest
+	containerMetadataExtension = criContainerdPrefix + ".container.metadata"
 )
 
 // makeSandboxName generates sandbox name from sandbox metadata. The name

pkg/server/sandbox_run.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/typeurl"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	"github.com/golang/glog"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -35,6 +36,11 @@ import (
 	"github.com/kubernetes-incubator/cri-containerd/pkg/util"
 )
 
+func init() {
+	typeurl.Register(&sandboxstore.Metadata{},
+		"github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox", "Metadata")
+}
+
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
 // the sandbox is in ready state.
 func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
@@ -117,15 +123,6 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	}
 	glog.V(4).Infof("Sandbox container spec: %+v", spec)
 
-	// Checkpoint metadata into container
-	metaBytes, err := sandbox.Metadata.Encode()
-	if err != nil {
-		return nil, fmt.Errorf("failed to convert sandbox metadata: %+v, %v", sandbox.Metadata, err)
-	}
-	labels := map[string]string{
-		sandboxMetadataLabel: string(metaBytes),
-	}
-
 	var specOpts []containerd.SpecOpts
 	if uid := config.GetLinux().GetSecurityContext().GetRunAsUser(); uid != nil {
 		specOpts = append(specOpts, containerd.WithUserID(uint32(uid.GetValue())))
@@ -134,7 +131,8 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		containerd.WithSnapshotter(c.config.ContainerdSnapshotter),
 		containerd.WithNewSnapshot(id, image.Image),
 		containerd.WithSpec(spec, specOpts...),
-		containerd.WithContainerLabels(labels),
+		containerd.WithContainerLabels(map[string]string{containerKindLabel: containerKindSandbox}),
+		containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
 		containerd.WithRuntime(defaultRuntime, nil)}
 	container, err := c.client.NewContainer(ctx, id, opts...)
 	if err != nil {

pkg/server/sandbox_run_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/containerd/containerd/typeurl"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
@@ -28,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
@@ -383,5 +385,45 @@ func TestToCNIPortMappings(t *testing.T) {
 	}
 }
 
+func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
+	for desc, test := range map[string]struct {
+		configChange func(*runtime.PodSandboxConfig)
+	}{
+		"should marshal original config": {},
+		"should marshal Linux": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						HostNetwork: true,
+						HostPid:     true,
+						HostIpc:     true,
+					},
+					SupplementalGroups: []int64{1111, 2222},
+				}
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		meta := &sandboxstore.Metadata{
+			ID:        "1",
+			Name:      "sandbox_1",
+			NetNSPath: "/home/cloud",
+		}
+		meta.Config, _, _ = getRunPodSandboxTestData()
+		if test.configChange != nil {
+			test.configChange(meta.Config)
+		}
+
+		any, err := typeurl.MarshalAny(meta)
+		assert.NoError(t, err)
+		data, err := typeurl.UnmarshalAny(any)
+		assert.NoError(t, err)
+		assert.IsType(t, &sandboxstore.Metadata{}, data)
+		curMeta, ok := data.(*sandboxstore.Metadata)
+		assert.True(t, ok)
+		assert.Equal(t, meta, curMeta)
+	}
+}
+
 // TODO(random-liu): [P1] Add unit test for different error cases to make sure
 // the function cleans up on error properly.

pkg/store/container/metadata.go

@@ -35,9 +35,13 @@ const metadataVersion = "v1" // nolint
 type versionedMetadata struct {
 	// Version indicates the version of the versioned container metadata.
 	Version string
-	Metadata
+	// Metadata's type is metadataInternal. If not there will be a recursive call in MarshalJSON.
+	Metadata metadataInternal
 }
 
+// metadataInternal is for internal use.
+type metadataInternal Metadata
+
 // Metadata is the unversioned container metadata.
 type Metadata struct {
 	// ID is the container id.
@@ -52,16 +56,16 @@ type Metadata struct {
 	ImageRef string
 }
 
-// Encode encodes Metadata into bytes in json format.
+// MarshalJSON encodes Metadata into bytes in json format.
-func (c *Metadata) Encode() ([]byte, error) {
+func (c *Metadata) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&versionedMetadata{
 		Version:  metadataVersion,
-		Metadata: *c,
+		Metadata: metadataInternal(*c),
 	})
 }
 
-// Decode decodes Metadata from bytes.
+// UnmarshalJSON decodes Metadata from bytes.
-func (c *Metadata) Decode(data []byte) error {
+func (c *Metadata) UnmarshalJSON(data []byte) error {
 	versioned := &versionedMetadata{}
 	if err := json.Unmarshal(data, versioned); err != nil {
 		return err
@@ -69,8 +73,8 @@ func (c *Metadata) Decode(data []byte) error {
 	// Handle old version after upgrade.
 	switch versioned.Version {
 	case metadataVersion:
-		*c = versioned.Metadata
+		*c = Metadata(versioned.Metadata)
 		return nil
 	}
-	return fmt.Errorf("unsupported version")
+	return fmt.Errorf("unsupported version: %q", versioned.Version)
 }

pkg/store/container/metadata_test.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-func TestMetadataEncodeDecode(t *testing.T) {
+func TestMetadataMarshalUnmarshal(t *testing.T) {
 	meta := &Metadata{
 		ID:        "test-id",
 		Name:      "test-name",
@@ -37,17 +37,44 @@ func TestMetadataEncodeDecode(t *testing.T) {
 		},
 		ImageRef: "test-image-ref",
 	}
-	assert := assertlib.New(t)
-	data, err := meta.Encode()
-	assert.NoError(err)
-	newMeta := &Metadata{}
-	assert.NoError(newMeta.Decode(data))
-	assert.Equal(meta, newMeta)
 
-	unsupported, err := json.Marshal(&versionedMetadata{
+	assert := assertlib.New(t)
-		Version:  "random-test-version",
+	newMeta := &Metadata{}
-		Metadata: *meta,
+	newVerMeta := &versionedMetadata{}
+
+	t.Logf("should be able to do json.marshal")
+	data, err := json.Marshal(meta)
+	assert.NoError(err)
+	data1, err := json.Marshal(&versionedMetadata{
+		Version:  metadataVersion,
+		Metadata: metadataInternal(*meta),
 	})
 	assert.NoError(err)
-	assert.Error(newMeta.Decode(unsupported))
+	assert.Equal(data, data1)
+
+	t.Logf("should be able to do MarshalJSON")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should be able to do MarshalJSON and json.Unmarshal")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(json.Unmarshal(data, newVerMeta))
+	assert.Equal(meta, (*Metadata)(&newVerMeta.Metadata))
+
+	t.Logf("should be able to do json.Marshal and UnmarshalJSON")
+	data, err = json.Marshal(meta)
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should json.Unmarshal fail for unsupported version")
+	unsupported, err := json.Marshal(&versionedMetadata{
+		Version:  "random-test-version",
+		Metadata: metadataInternal(*meta),
+	})
+	assert.NoError(err)
+	assert.Error(json.Unmarshal(unsupported, &newMeta))
 }

pkg/store/sandbox/metadata.go

@@ -35,9 +35,13 @@ const metadataVersion = "v1" // nolint
 type versionedMetadata struct {
 	// Version indicates the version of the versioned sandbox metadata.
 	Version string
-	Metadata
+	// Metadata's type is metadataInternal. If not there will be a recursive call in MarshalJSON.
+	Metadata metadataInternal
 }
 
+// metadataInternal is for internal use.
+type metadataInternal Metadata
+
 // Metadata is the unversioned sandbox metadata.
 type Metadata struct {
 	// ID is the sandbox id.
@@ -50,16 +54,16 @@ type Metadata struct {
 	NetNSPath string
 }
 
-// Encode encodes Metadata into bytes in json format.
+// MarshalJSON encodes Metadata into bytes in json format.
-func (c *Metadata) Encode() ([]byte, error) {
+func (c *Metadata) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&versionedMetadata{
 		Version:  metadataVersion,
-		Metadata: *c,
+		Metadata: metadataInternal(*c),
 	})
 }
 
-// Decode decodes Metadata from bytes.
+// UnmarshalJSON decodes Metadata from bytes.
-func (c *Metadata) Decode(data []byte) error {
+func (c *Metadata) UnmarshalJSON(data []byte) error {
 	versioned := &versionedMetadata{}
 	if err := json.Unmarshal(data, versioned); err != nil {
 		return err
@@ -67,8 +71,8 @@ func (c *Metadata) Decode(data []byte) error {
 	// Handle old version after upgrade.
 	switch versioned.Version {
 	case metadataVersion:
-		*c = versioned.Metadata
+		*c = Metadata(versioned.Metadata)
 		return nil
 	}
-	return fmt.Errorf("unsupported version")
+	return fmt.Errorf("unsupported version: %q", versioned.Version)
 }

pkg/store/sandbox/metadata_test.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-func TestMetadataEncodeDecode(t *testing.T) {
+func TestMetadataMarshalUnmarshal(t *testing.T) {
 	meta := &Metadata{
 		ID:   "test-id",
 		Name: "test-name",
@@ -38,16 +38,42 @@ func TestMetadataEncodeDecode(t *testing.T) {
 		},
 	}
 	assert := assertlib.New(t)
-	data, err := meta.Encode()
-	assert.NoError(err)
 	newMeta := &Metadata{}
-	assert.NoError(newMeta.Decode(data))
+	newVerMeta := &versionedMetadata{}
-	assert.Equal(meta, newMeta)
 
-	unsupported, err := json.Marshal(&versionedMetadata{
+	t.Logf("should be able to do json.marshal")
-		Version:  "random-test-version",
+	data, err := json.Marshal(meta)
-		Metadata: *meta,
+	assert.NoError(err)
+	data1, err := json.Marshal(&versionedMetadata{
+		Version:  metadataVersion,
+		Metadata: metadataInternal(*meta),
 	})
 	assert.NoError(err)
-	assert.Error(newMeta.Decode(unsupported))
+	assert.Equal(data, data1)
+
+	t.Logf("should be able to do MarshalJSON")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should be able to do MarshalJSON and json.Unmarshal")
+	data, err = meta.MarshalJSON()
+	assert.NoError(err)
+	assert.NoError(json.Unmarshal(data, newVerMeta))
+	assert.Equal(meta, (*Metadata)(&newVerMeta.Metadata))
+
+	t.Logf("should be able to do json.Marshal and UnmarshalJSON")
+	data, err = json.Marshal(meta)
+	assert.NoError(err)
+	assert.NoError(newMeta.UnmarshalJSON(data))
+	assert.Equal(meta, newMeta)
+
+	t.Logf("should json.Unmarshal fail for unsupported version")
+	unsupported, err := json.Marshal(&versionedMetadata{
+		Version:  "random-test-version",
+		Metadata: metadataInternal(*meta),
+	})
+	assert.NoError(err)
+	assert.Error(json.Unmarshal(unsupported, &newMeta))
 }