Use new metadata store.

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/server/container_create.go

@@ -32,7 +32,7 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // CreateContainer creates a new container in the given PodSandbox.
@@ -58,7 +58,7 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 	// the same container.
 	id := generateID()
 	name := makeContainerName(config.GetMetadata(), sandboxConfig.GetMetadata())
-	if err := c.containerNameIndex.Reserve(name, id); err != nil {
+	if err = c.containerNameIndex.Reserve(name, id); err != nil {
 		return nil, fmt.Errorf("failed to reserve container name %q: %v", name, err)
 	}
 	defer func() {
@@ -68,8 +68,8 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 		}
 	}()
 
-	// Create initial container metadata.
+	// Create initial internal container metadata.
-	meta := metadata.ContainerMetadata{
+	meta := containerstore.Metadata{
 		ID:        id,
 		Name:      name,
 		SandboxID: sandboxID,
@@ -78,18 +78,18 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 
 	// Prepare container image snapshot. For container, the image should have
 	// been pulled before creating the container, so do not ensure the image.
-	image := config.GetImage().GetImage()
+	imageRef := config.GetImage().GetImage()
-	imageMeta, err := c.localResolve(ctx, image)
+	image, err := c.localResolve(ctx, imageRef)
 	if err != nil {
-		return nil, fmt.Errorf("failed to resolve image %q: %v", image, err)
+		return nil, fmt.Errorf("failed to resolve image %q: %v", imageRef, err)
 	}
-	if imageMeta == nil {
+	if image == nil {
-		return nil, fmt.Errorf("image %q not found", image)
+		return nil, fmt.Errorf("image %q not found", imageRef)
 	}
 
 	// Generate container runtime spec.
 	mounts := c.generateContainerMounts(getSandboxRootDir(c.rootDir, sandboxID), config)
-	spec, err := c.generateContainerSpec(id, sandbox.Pid, config, sandboxConfig, imageMeta.Config, mounts)
+	spec, err := c.generateContainerSpec(id, sandbox.Pid, config, sandboxConfig, image.Config, mounts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate container %q spec: %v", id, err)
 	}
@@ -101,12 +101,12 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 
 	// Prepare container rootfs.
 	if config.GetLinux().GetSecurityContext().GetReadonlyRootfs() {
-		if _, err := c.snapshotService.View(ctx, id, imageMeta.ChainID); err != nil {
+		if _, err := c.snapshotService.View(ctx, id, image.ChainID); err != nil {
-			return nil, fmt.Errorf("failed to view container rootfs %q: %v", imageMeta.ChainID, err)
+			return nil, fmt.Errorf("failed to view container rootfs %q: %v", image.ChainID, err)
 		}
 	} else {
-		if _, err := c.snapshotService.Prepare(ctx, id, imageMeta.ChainID); err != nil {
+		if _, err := c.snapshotService.Prepare(ctx, id, image.ChainID); err != nil {
-			return nil, fmt.Errorf("failed to prepare container rootfs %q: %v", imageMeta.ChainID, err)
+			return nil, fmt.Errorf("failed to prepare container rootfs %q: %v", image.ChainID, err)
 		}
 	}
 	defer func() {
@@ -116,18 +116,18 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 			}
 		}
 	}()
-	meta.ImageRef = imageMeta.ID
+	meta.ImageRef = image.ID
 
 	// Create container root directory.
 	containerRootDir := getContainerRootDir(c.rootDir, id)
-	if err := c.os.MkdirAll(containerRootDir, 0755); err != nil {
+	if err = c.os.MkdirAll(containerRootDir, 0755); err != nil {
 		return nil, fmt.Errorf("failed to create container root directory %q: %v",
 			containerRootDir, err)
 	}
 	defer func() {
 		if retErr != nil {
 			// Cleanup the container root directory.
-			if err := c.os.RemoveAll(containerRootDir); err != nil {
+			if err = c.os.RemoveAll(containerRootDir); err != nil {
 				glog.Errorf("Failed to remove container root directory %q: %v",
 					containerRootDir, err)
 			}
@@ -139,7 +139,7 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 		Container: containers.Container{
 			ID: id,
 			// TODO(random-liu): Checkpoint metadata into container labels.
-			Image:   imageMeta.ID,
+			Image:   image.ID,
 			Runtime: defaultRuntime,
 			Spec: &prototypes.Any{
 				TypeUrl: runtimespec.Version,
@@ -158,12 +158,23 @@ func (c *criContainerdService) CreateContainer(ctx context.Context, r *runtime.C
 		}
 	}()
 
-	// Update container CreatedAt.
+	container, err := containerstore.NewContainer(meta, containerstore.Status{CreatedAt: time.Now().UnixNano()})
-	meta.CreatedAt = time.Now().UnixNano()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create internal container object for %q: %v",
+			id, err)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup container checkpoint on error.
+			if err := container.Delete(); err != nil {
+				glog.Errorf("Failed to cleanup container checkpoint for %q: %v", id, err)
+			}
+		}
+	}()
+
 	// Add container into container store.
-	if err := c.containerStore.Create(meta); err != nil {
+	if err := c.containerStore.Add(container); err != nil {
-		return nil, fmt.Errorf("failed to add container metadata %+v into store: %v",
+		return nil, fmt.Errorf("failed to add container %q into store: %v", id, err)
-			meta, err)
 	}
 
 	return &runtime.CreateContainerResponse{ContainerId: id}, nil

pkg/server/container_create_test.go

@@ -32,9 +32,11 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string,
@@ -443,64 +445,66 @@ func TestCreateContainer(t *testing.T) {
 	testSandboxID := "test-sandbox-id"
 	testSandboxPid := uint32(4321)
 	config, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
-	testSandboxMetadata := &metadata.SandboxMetadata{
+	testSandbox := &sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:     testSandboxID,
 			Name:   "test-sandbox-name",
 			Config: sandboxConfig,
 			Pid:    testSandboxPid,
+		},
 	}
 	testContainerName := makeContainerName(config.Metadata, sandboxConfig.Metadata)
 	// Use an image id to avoid image name resolution.
 	// TODO(random-liu): Change this to image name after we have complete image
 	// management unit test framework.
-	testImage := config.GetImage().GetImage()
+	testImageRef := config.GetImage().GetImage()
 	testChainID := "test-chain-id"
-	testImageMetadata := metadata.ImageMetadata{
+	testImage := imagestore.Image{
-		ID:      testImage,
+		ID:      testImageRef,
 		ChainID: testChainID,
 		Config:  imageConfig,
 	}
 
 	for desc, test := range map[string]struct {
-		sandboxMetadata    *metadata.SandboxMetadata
+		sandbox            *sandboxstore.Sandbox
 		reserveNameErr     bool
-		imageMetadataErr   bool
+		imageStoreErr      bool
 		prepareSnapshotErr error
 		createRootDirErr   error
 		expectErr          bool
-		expectMeta         *metadata.ContainerMetadata
+		expectedMeta       containerstore.Metadata
 	}{
 		"should return error if sandbox does not exist": {
-			sandboxMetadata: nil,
+			sandbox:   nil,
 			expectErr: true,
 		},
 		"should return error if name is reserved": {
-			sandboxMetadata: testSandboxMetadata,
+			sandbox:        testSandbox,
 			reserveNameErr: true,
 			expectErr:      true,
 		},
 		"should return error if fail to create root directory": {
-			sandboxMetadata:  testSandboxMetadata,
+			sandbox:          testSandbox,
 			createRootDirErr: errors.New("random error"),
 			expectErr:        true,
 		},
 		"should return error if image is not pulled": {
-			sandboxMetadata:  testSandboxMetadata,
+			sandbox:       testSandbox,
-			imageMetadataErr: true,
+			imageStoreErr: true,
 			expectErr:     true,
 		},
 		"should return error if prepare snapshot fails": {
-			sandboxMetadata:    testSandboxMetadata,
+			sandbox:            testSandbox,
 			prepareSnapshotErr: errors.New("random error"),
 			expectErr:          true,
 		},
 		"should be able to create container successfully": {
-			sandboxMetadata: testSandboxMetadata,
+			sandbox:   testSandbox,
 			expectErr: false,
-			expectMeta: &metadata.ContainerMetadata{
+			expectedMeta: containerstore.Metadata{
 				Name:      testContainerName,
 				SandboxID: testSandboxID,
-				ImageRef:  testImage,
+				ImageRef:  testImageRef,
 				Config:    config,
 			},
 		},
@@ -510,14 +514,14 @@ func TestCreateContainer(t *testing.T) {
 		fake := c.containerService.(*servertesting.FakeContainersClient)
 		fakeSnapshotClient := WithFakeSnapshotClient(c)
 		fakeOS := c.os.(*ostesting.FakeOS)
-		if test.sandboxMetadata != nil {
+		if test.sandbox != nil {
-			assert.NoError(t, c.sandboxStore.Create(*test.sandboxMetadata))
+			assert.NoError(t, c.sandboxStore.Add(*test.sandbox))
 		}
 		if test.reserveNameErr {
 			assert.NoError(t, c.containerNameIndex.Reserve(testContainerName, "random id"))
 		}
-		if !test.imageMetadataErr {
+		if !test.imageStoreErr {
-			assert.NoError(t, c.imageMetadataStore.Create(testImageMetadata))
+			c.imageStore.Add(testImage)
 		}
 		if test.prepareSnapshotErr != nil {
 			fakeSnapshotClient.InjectError("prepare", test.prepareSnapshotErr)
@@ -554,9 +558,7 @@ func TestCreateContainer(t *testing.T) {
 			listResp, err := fake.List(context.Background(), &containers.ListContainersRequest{})
 			assert.NoError(t, err)
 			assert.Empty(t, listResp.Containers, "containerd container should be cleaned up")
-			metas, err := c.containerStore.List()
+			assert.Empty(t, c.containerStore.List(), "container metadata should not be created")
-			assert.NoError(t, err)
-			assert.Empty(t, metas, "container metadata should not be created")
 			continue
 		}
 		assert.NoError(t, err)
@@ -571,7 +573,7 @@ func TestCreateContainer(t *testing.T) {
 		createOpts, ok := containersCalls[0].Argument.(*containers.CreateContainerRequest)
 		assert.True(t, ok, "should create containerd container")
 		assert.Equal(t, id, createOpts.Container.ID, "container id should be correct")
-		assert.Equal(t, testImage, createOpts.Container.Image, "test image should be correct")
+		assert.Equal(t, testImageRef, createOpts.Container.Image, "test image should be correct")
 		assert.Equal(t, id, createOpts.Container.RootFS, "rootfs should be correct")
 		spec := &runtimespec.Spec{}
 		assert.NoError(t, json.Unmarshal(createOpts.Container.Spec.Value, spec))
@@ -586,12 +588,11 @@ func TestCreateContainer(t *testing.T) {
 			Parent: testChainID,
 		}, prepareOpts, "prepare request should be correct")
 
-		meta, err := c.containerStore.Get(id)
+		container, err := c.containerStore.Get(id)
 		assert.NoError(t, err)
-		require.NotNil(t, meta)
+		test.expectedMeta.ID = id
-		test.expectMeta.ID = id
+		assert.Equal(t, test.expectedMeta, container.Metadata, "container metadata should be created")
-		// TODO(random-liu): Use fake clock to test CreatedAt.
+		assert.Equal(t, runtime.ContainerState_CONTAINER_CREATED, container.Status.Get().State(),
-		test.expectMeta.CreatedAt = meta.CreatedAt
+			"container should be in created state")
-		assert.Equal(t, test.expectMeta, meta, "container metadata should be created")
 	}
 }

pkg/server/container_execsync.go

@@ -45,15 +45,16 @@ func (c *criContainerdService) ExecSync(ctx context.Context, r *runtime.ExecSync
 		}
 	}()
 
-	// Get container metadata from our container store.
+	// Get container from our container store.
-	meta, err := c.containerStore.Get(r.GetContainerId())
+	cntr, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("an error occurred when try to find container %q: %v", r.GetContainerId(), err)
 	}
-	id := meta.ID
+	id := cntr.ID
 
-	if meta.State() != runtime.ContainerState_CONTAINER_RUNNING {
+	state := cntr.Status.Get().State()
-		return nil, fmt.Errorf("container %q is in %s state", id, criContainerStateToString(meta.State()))
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, fmt.Errorf("container %q is in %s state", id, criContainerStateToString(state))
 	}
 
 	// Get exec process spec.

pkg/server/container_list.go

@@ -17,14 +17,12 @@ limitations under the License.
 package server
 
 import (
-	"fmt"
-
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // ListContainers lists all containers matching the filter.
@@ -36,33 +34,31 @@ func (c *criContainerdService) ListContainers(ctx context.Context, r *runtime.Li
 		}
 	}()
 
-	// List all container metadata from store.
+	// List all containers from store.
-	metas, err := c.containerStore.List()
+	containersInStore := c.containerStore.List()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list metadata from container store: %v", err)
-	}
 
 	var containers []*runtime.Container
-	for _, meta := range metas {
+	for _, container := range containersInStore {
-		containers = append(containers, toCRIContainer(meta))
+		containers = append(containers, toCRIContainer(container))
 	}
 
 	containers = c.filterCRIContainers(containers, r.GetFilter())
 	return &runtime.ListContainersResponse{Containers: containers}, nil
 }
 
-// toCRIContainer converts container metadata into CRI container.
+// toCRIContainer converts internal container object into CRI container.
-func toCRIContainer(meta *metadata.ContainerMetadata) *runtime.Container {
+func toCRIContainer(container containerstore.Container) *runtime.Container {
+	status := container.Status.Get()
 	return &runtime.Container{
-		Id:           meta.ID,
+		Id:           container.ID,
-		PodSandboxId: meta.SandboxID,
+		PodSandboxId: container.SandboxID,
-		Metadata:     meta.Config.GetMetadata(),
+		Metadata:     container.Config.GetMetadata(),
-		Image:        meta.Config.GetImage(),
+		Image:        container.Config.GetImage(),
-		ImageRef:     meta.ImageRef,
+		ImageRef:     container.ImageRef,
-		State:        meta.State(),
+		State:        status.State(),
-		CreatedAt:    meta.CreatedAt,
+		CreatedAt:    status.CreatedAt,
-		Labels:       meta.Config.GetLabels(),
+		Labels:       container.Config.GetLabels(),
-		Annotations:  meta.Config.GetAnnotations(),
+		Annotations:  container.Config.GetAnnotations(),
 	}
 }
 

pkg/server/container_list_test.go

@@ -23,10 +23,9 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
-
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 func TestToCRIContainer(t *testing.T) {
@@ -40,12 +39,15 @@ func TestToCRIContainer(t *testing.T) {
 		Annotations: map[string]string{"c": "d"},
 	}
 	createdAt := time.Now().UnixNano()
-	meta := &metadata.ContainerMetadata{
+	container, err := containerstore.NewContainer(
+		containerstore.Metadata{
 			ID:        "test-id",
 			Name:      "test-name",
 			SandboxID: "test-sandbox-id",
 			Config:    config,
 			ImageRef:  "test-image-ref",
+		},
+		containerstore.Status{
 			Pid:        1234,
 			CreatedAt:  createdAt,
 			StartedAt:  time.Now().UnixNano(),
@@ -53,7 +55,9 @@ func TestToCRIContainer(t *testing.T) {
 			ExitCode:   1,
 			Reason:     "test-reason",
 			Message:    "test-message",
-	}
+		},
+	)
+	assert.NoError(t, err)
 	expect := &runtime.Container{
 		Id:           "test-id",
 		PodSandboxId: "test-sandbox-id",
@@ -65,7 +69,7 @@ func TestToCRIContainer(t *testing.T) {
 		Labels:       config.GetLabels(),
 		Annotations:  config.GetAnnotations(),
 	}
-	c := toCRIContainer(meta)
+	c := toCRIContainer(container)
 	assert.Equal(t, expect, c)
 }
 
@@ -147,44 +151,68 @@ func TestFilterContainers(t *testing.T) {
 	}
 }
 
+// containerForTest is a helper type for test.
+type containerForTest struct {
+	metadata containerstore.Metadata
+	status   containerstore.Status
+}
+
+func (c containerForTest) toContainer() (containerstore.Container, error) {
+	return containerstore.NewContainer(c.metadata, c.status)
+}
+
 func TestListContainers(t *testing.T) {
 	c := newTestCRIContainerdService()
 
 	createdAt := time.Now().UnixNano()
 	startedAt := time.Now().UnixNano()
 	finishedAt := time.Now().UnixNano()
-	containersInStore := []metadata.ContainerMetadata{
+	containersInStore := []containerForTest{
 		{
+			metadata: containerstore.Metadata{
 				ID:        "1",
 				Name:      "name-1",
 				SandboxID: "s-1",
 				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-1"}},
-			CreatedAt: createdAt,
+			},
+			status: containerstore.Status{CreatedAt: createdAt},
 		},
 		{
+			metadata: containerstore.Metadata{
 				ID:        "2",
 				Name:      "name-2",
 				SandboxID: "s-1",
 				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-2"}},
+			},
+			status: containerstore.Status{
 				CreatedAt: createdAt,
 				StartedAt: startedAt,
 			},
+		},
 		{
+			metadata: containerstore.Metadata{
 				ID:        "3",
 				Name:      "name-3",
 				SandboxID: "s-1",
 				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-3"}},
+			},
+			status: containerstore.Status{
 				CreatedAt:  createdAt,
 				StartedAt:  startedAt,
 				FinishedAt: finishedAt,
 			},
+		},
 		{
+			metadata: containerstore.Metadata{
 				ID:        "4",
 				Name:      "name-4",
 				SandboxID: "s-2",
 				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-4"}},
+			},
+			status: containerstore.Status{
 				CreatedAt: createdAt,
 			},
+		},
 	}
 	filter := &runtime.ContainerFilter{
 		PodSandboxId: "s-1",
@@ -215,7 +243,9 @@ func TestListContainers(t *testing.T) {
 
 	// Inject test metadata
 	for _, cntr := range containersInStore {
-		c.containerStore.Create(cntr)
+		container, err := cntr.toContainer()
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(container))
 	}
 
 	resp, err := c.ListContainers(context.Background(), &runtime.ListContainersRequest{Filter: filter})

pkg/server/container_remove.go

@@ -25,7 +25,8 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // RemoveContainer removes the container.
@@ -37,31 +38,29 @@ func (c *criContainerdService) RemoveContainer(ctx context.Context, r *runtime.R
 		}
 	}()
 
-	id := r.GetContainerId()
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, fmt.Errorf("an error occurred when try to find container %q: %v", r.GetContainerId(), err)
+		}
+		// Do not return error if container metadata doesn't exist.
+		glog.V(5).Infof("RemoveContainer called for container %q that does not exist", r.GetContainerId())
+		return &runtime.RemoveContainerResponse{}, nil
+	}
+	id := container.ID
 
 	// Set removing state to prevent other start/remove operations against this container
 	// while it's being removed.
-	if err := c.setContainerRemoving(id); err != nil {
+	if err := setContainerRemoving(container); err != nil {
-		if !metadata.IsNotExistError(err) {
+		return nil, fmt.Errorf("failed to set removing state for container %q: %v", id, err)
-			return nil, fmt.Errorf("failed to set removing state for container %q: %v",
-				id, err)
-		}
-		// Do not return error if container metadata doesn't exist.
-		glog.V(5).Infof("RemoveContainer called for container %q that does not exist", id)
-		return &runtime.RemoveContainerResponse{}, nil
 	}
 	defer func() {
-		if retErr == nil {
+		if retErr != nil {
-			// Cleanup all index after successfully remove the container.
-			c.containerNameIndex.ReleaseByKey(id)
-			return
-		}
 			// Reset removing if remove failed.
-		if err := c.resetContainerRemoving(id); err != nil {
+			if err := resetContainerRemoving(container); err != nil {
-			// TODO(random-liu): Deal with update failure. Actually Removing doesn't need to
+				// TODO(random-liu): Do not checkpoint `Removing` state.
-			// be checkpointed, we only need it to have the same lifecycle with container metadata.
+				glog.Errorf("failed to reset removing state for container %q: %v", id, err)
-			glog.Errorf("failed to reset removing state for container %q: %v",
+			}
-				id, err)
 		}
 	}()
 
@@ -78,13 +77,17 @@ func (c *criContainerdService) RemoveContainer(ctx context.Context, r *runtime.R
 		glog.V(5).Infof("Remove called for snapshot %q that does not exist", id)
 	}
 
-	// Cleanup container root directory.
 	containerRootDir := getContainerRootDir(c.rootDir, id)
 	if err := c.os.RemoveAll(containerRootDir); err != nil {
 		return nil, fmt.Errorf("failed to remove container root directory %q: %v",
 			containerRootDir, err)
 	}
 
+	// Delete container checkpoint.
+	if err := container.Delete(); err != nil {
+		return nil, fmt.Errorf("failed to delete container checkpoint for %q: %v", id, err)
+	}
+
 	// Delete containerd container.
 	if _, err := c.containerService.Delete(ctx, &containers.DeleteContainerRequest{ID: id}); err != nil {
 		if !isContainerdGRPCNotFoundError(err) {
@@ -93,35 +96,34 @@ func (c *criContainerdService) RemoveContainer(ctx context.Context, r *runtime.R
 		glog.V(5).Infof("Remove called for containerd container %q that does not exist", id, err)
 	}
 
-	// Delete container metadata.
+	c.containerStore.Delete(id)
-	if err := c.containerStore.Delete(id); err != nil {
+
-		return nil, fmt.Errorf("failed to delete container metadata for %q: %v", id, err)
+	c.containerNameIndex.ReleaseByKey(id)
-	}
 
 	return &runtime.RemoveContainerResponse{}, nil
 }
 
 // setContainerRemoving sets the container into removing state. In removing state, the
 // container will not be started or removed again.
-func (c *criContainerdService) setContainerRemoving(id string) error {
+func setContainerRemoving(container containerstore.Container) error {
-	return c.containerStore.Update(id, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
 		// Do not remove container if it's still running.
-		if meta.State() == runtime.ContainerState_CONTAINER_RUNNING {
+		if status.State() == runtime.ContainerState_CONTAINER_RUNNING {
-			return meta, fmt.Errorf("container %q is still running", id)
+			return status, fmt.Errorf("container is still running")
 		}
-		if meta.Removing {
+		if status.Removing {
-			return meta, fmt.Errorf("container is already in removing state")
+			return status, fmt.Errorf("container is already in removing state")
 		}
-		meta.Removing = true
+		status.Removing = true
-		return meta, nil
+		return status, nil
 	})
 }
 
 // resetContainerRemoving resets the container removing state on remove failure. So
 // that we could remove the container again.
-func (c *criContainerdService) resetContainerRemoving(id string) error {
+func resetContainerRemoving(container containerstore.Container) error {
-	return c.containerStore.Update(id, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
-		meta.Removing = false
+		status.Removing = false
-		return meta, nil
+		return status, nil
 	})
 }

pkg/server/container_remove_test.go

@@ -25,13 +25,13 @@ import (
 	snapshotapi "github.com/containerd/containerd/api/services/snapshot"
 	"github.com/containerd/containerd/api/types/mount"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // TestSetContainerRemoving tests setContainerRemoving sets removing
@@ -39,20 +39,18 @@ import (
 func TestSetContainerRemoving(t *testing.T) {
 	testID := "test-id"
 	for desc, test := range map[string]struct {
-		metadata  *metadata.ContainerMetadata
+		status    containerstore.Status
 		expectErr bool
 	}{
 		"should return error when container is in running state": {
-			metadata: &metadata.ContainerMetadata{
+			status: containerstore.Status{
-				ID:        testID,
 				CreatedAt: time.Now().UnixNano(),
 				StartedAt: time.Now().UnixNano(),
 			},
 			expectErr: true,
 		},
 		"should return error when container is in removing state": {
-			metadata: &metadata.ContainerMetadata{
+			status: containerstore.Status{
-				ID:         testID,
 				CreatedAt:  time.Now().UnixNano(),
 				StartedAt:  time.Now().UnixNano(),
 				FinishedAt: time.Now().UnixNano(),
@@ -61,8 +59,7 @@ func TestSetContainerRemoving(t *testing.T) {
 			expectErr: true,
 		},
 		"should not return error when container is not running and removing": {
-			metadata: &metadata.ContainerMetadata{
+			status: containerstore.Status{
-				ID:         testID,
 				CreatedAt:  time.Now().UnixNano(),
 				StartedAt:  time.Now().UnixNano(),
 				FinishedAt: time.Now().UnixNano(),
@@ -71,19 +68,18 @@ func TestSetContainerRemoving(t *testing.T) {
 		},
 	} {
 		t.Logf("TestCase %q", desc)
-		c := newTestCRIContainerdService()
+		container, err := containerstore.NewContainer(
-		if test.metadata != nil {
+			containerstore.Metadata{ID: testID},
-			assert.NoError(t, c.containerStore.Create(*test.metadata))
+			test.status,
-		}
+		)
-		err := c.setContainerRemoving(testID)
+		assert.NoError(t, err)
-		meta, getErr := c.containerStore.Get(testID)
+		err = setContainerRemoving(container)
-		assert.NoError(t, getErr)
 		if test.expectErr {
 			assert.Error(t, err)
-			assert.Equal(t, test.metadata, meta, "metadata should not be updated")
+			assert.Equal(t, test.status, container.Status.Get(), "metadata should not be updated")
 		} else {
 			assert.NoError(t, err)
-			assert.True(t, meta.Removing, "removing should be set")
+			assert.True(t, container.Status.Get().Removing, "removing should be set")
 		}
 	}
 }
@@ -91,15 +87,14 @@ func TestSetContainerRemoving(t *testing.T) {
 func TestRemoveContainer(t *testing.T) {
 	testID := "test-id"
 	testName := "test-name"
-	testContainerMetadata := &metadata.ContainerMetadata{
+	testContainerStatus := &containerstore.Status{
-		ID:         testID,
 		CreatedAt:  time.Now().UnixNano(),
 		StartedAt:  time.Now().UnixNano(),
 		FinishedAt: time.Now().UnixNano(),
 	}
 
 	for desc, test := range map[string]struct {
-		metadata            *metadata.ContainerMetadata
+		status              *containerstore.Status
 		removeSnapshotErr   error
 		deleteContainerErr  error
 		removeDirErr        error
@@ -107,16 +102,14 @@ func TestRemoveContainer(t *testing.T) {
 		expectUnsetRemoving bool
 	}{
 		"should return error when container is still running": {
-			metadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:        testID,
 				CreatedAt: time.Now().UnixNano(),
 				StartedAt: time.Now().UnixNano(),
 			},
 			expectErr: true,
 		},
 		"should return error when there is ongoing removing": {
-			metadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:         testID,
 				CreatedAt:  time.Now().UnixNano(),
 				StartedAt:  time.Now().UnixNano(),
 				FinishedAt: time.Now().UnixNano(),
@@ -124,40 +117,40 @@ func TestRemoveContainer(t *testing.T) {
 			},
 			expectErr: true,
 		},
-		"should not return error if container metadata does not exist": {
+		"should not return error if container does not exist": {
-			metadata:           nil,
+			status:             nil,
 			removeSnapshotErr:  servertesting.SnapshotNotExistError,
 			deleteContainerErr: servertesting.ContainerNotExistError,
 			expectErr:          false,
 		},
 		"should not return error if snapshot does not exist": {
-			metadata:          testContainerMetadata,
+			status:            testContainerStatus,
 			removeSnapshotErr: servertesting.SnapshotNotExistError,
 			expectErr:         false,
 		},
 		"should return error if remove snapshot fails": {
-			metadata:          testContainerMetadata,
+			status:            testContainerStatus,
 			removeSnapshotErr: errors.New("random error"),
 			expectErr:         true,
 		},
 		"should not return error if containerd container does not exist": {
-			metadata:           testContainerMetadata,
+			status:             testContainerStatus,
 			deleteContainerErr: servertesting.ContainerNotExistError,
 			expectErr:          false,
 		},
 		"should return error if delete containerd container fails": {
-			metadata:           testContainerMetadata,
+			status:             testContainerStatus,
 			deleteContainerErr: errors.New("random error"),
 			expectErr:          true,
 		},
 		"should return error if remove container root fails": {
-			metadata:            testContainerMetadata,
+			status:              testContainerStatus,
 			removeDirErr:        errors.New("random error"),
 			expectErr:           true,
 			expectUnsetRemoving: true,
 		},
 		"should be able to remove container successfully": {
-			metadata:  testContainerMetadata,
+			status:    testContainerStatus,
 			expectErr: false,
 		},
 	} {
@@ -166,9 +159,14 @@ func TestRemoveContainer(t *testing.T) {
 		fake := c.containerService.(*servertesting.FakeContainersClient)
 		fakeSnapshotClient := WithFakeSnapshotClient(c)
 		fakeOS := c.os.(*ostesting.FakeOS)
-		if test.metadata != nil {
+		if test.status != nil {
 			assert.NoError(t, c.containerNameIndex.Reserve(testName, testID))
-			assert.NoError(t, c.containerStore.Create(*test.metadata))
+			container, err := containerstore.NewContainer(
+				containerstore.Metadata{ID: testID},
+				*test.status,
+			)
+			assert.NoError(t, err)
+			assert.NoError(t, c.containerStore.Add(container))
 		}
 		fakeOS.RemoveAllFn = func(path string) error {
 			assert.Equal(t, getContainerRootDir(c.rootDir, testID), path)
@@ -202,19 +200,16 @@ func TestRemoveContainer(t *testing.T) {
 			if !test.expectUnsetRemoving {
 				continue
 			}
-			meta, err := c.containerStore.Get(testID)
+			container, err := c.containerStore.Get(testID)
 			assert.NoError(t, err)
-			require.NotNil(t, meta)
 			// Also covers resetContainerRemoving.
-			assert.False(t, meta.Removing, "removing state should be unset")
+			assert.False(t, container.Status.Get().Removing, "removing state should be unset")
 			continue
 		}
 		assert.NoError(t, err)
 		assert.NotNil(t, resp)
-		meta, err := c.containerStore.Get(testID)
+		_, err = c.containerStore.Get(testID)
-		assert.Error(t, err)
+		assert.Equal(t, store.ErrNotExist, err)
-		assert.True(t, metadata.IsNotExistError(err))
-		assert.Nil(t, meta, "container metadata should be removed")
 		assert.NoError(t, c.containerNameIndex.Reserve(testName, testID),
 			"container name should be released")
 		mountsResp, err := fakeSnapshotClient.Mounts(context.Background(), &snapshotapi.MountsRequest{Key: testID})
@@ -229,6 +224,5 @@ func TestRemoveContainer(t *testing.T) {
 		})
 		assert.NoError(t, err)
 		assert.NotNil(t, resp, "remove should be idempotent")
-
 	}
 }

pkg/server/container_start.go

@@ -30,8 +30,8 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/server/agents"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // StartContainer starts the container.
@@ -50,12 +50,12 @@ func (c *criContainerdService) StartContainer(ctx context.Context, r *runtime.St
 	id := container.ID
 
 	var startErr error
-	// start container in one transaction to avoid race with event monitor.
+	// update container status in one transaction to avoid race with event monitor.
-	if err := c.containerStore.Update(id, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
+	if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
-		// Always apply metadata change no matter startContainer fails or not. Because startContainer
+		// Always apply status change no matter startContainer fails or not. Because startContainer
 		// may change container state no matter it fails or succeeds.
-		startErr = c.startContainer(ctx, id, &meta)
+		startErr = c.startContainer(ctx, id, container.Metadata, &status)
-		return meta, nil
+		return status, nil
 	}); startErr != nil {
 		return nil, startErr
 	} else if err != nil {
@@ -65,36 +65,36 @@ func (c *criContainerdService) StartContainer(ctx context.Context, r *runtime.St
 }
 
 // startContainer actually starts the container. The function needs to be run in one transaction. Any updates
-// to the metadata passed in will be applied to container store no matter the function returns error or not.
+// to the status passed in will be applied no matter the function returns error or not.
-func (c *criContainerdService) startContainer(ctx context.Context, id string, meta *metadata.ContainerMetadata) (retErr error) {
+func (c *criContainerdService) startContainer(ctx context.Context, id string, meta containerstore.Metadata, status *containerstore.Status) (retErr error) {
 	config := meta.Config
 	// Return error if container is not in created state.
-	if meta.State() != runtime.ContainerState_CONTAINER_CREATED {
+	if status.State() != runtime.ContainerState_CONTAINER_CREATED {
-		return fmt.Errorf("container %q is in %s state", id, criContainerStateToString(meta.State()))
+		return fmt.Errorf("container %q is in %s state", id, criContainerStateToString(status.State()))
 	}
 
 	// Do not start the container when there is a removal in progress.
-	if meta.Removing {
+	if status.Removing {
 		return fmt.Errorf("container %q is in removing state", id)
 	}
 
 	defer func() {
 		if retErr != nil {
 			// Set container to exited if fail to start.
-			meta.Pid = 0
+			status.Pid = 0
-			meta.FinishedAt = time.Now().UnixNano()
+			status.FinishedAt = time.Now().UnixNano()
-			meta.ExitCode = errorStartExitCode
+			status.ExitCode = errorStartExitCode
-			meta.Reason = errorStartReason
+			status.Reason = errorStartReason
-			meta.Message = retErr.Error()
+			status.Message = retErr.Error()
 		}
 	}()
 
 	// Get sandbox config from sandbox store.
-	sandboxMeta, err := c.sandboxStore.Get(meta.SandboxID)
+	sandbox, err := c.sandboxStore.Get(meta.SandboxID)
 	if err != nil {
 		return fmt.Errorf("sandbox %q not found: %v", meta.SandboxID, err)
 	}
-	sandboxConfig := sandboxMeta.Config
+	sandboxConfig := sandbox.Config
 	sandboxID := meta.SandboxID
 	// Make sure sandbox is running.
 	sandboxInfo, err := c.taskService.Info(ctx, &execution.InfoRequest{ContainerID: sandboxID})
@@ -137,12 +137,12 @@ func (c *criContainerdService) startContainer(ctx context.Context, id string, me
 	if config.GetLogPath() != "" {
 		// Only generate container log when log path is specified.
 		logPath := filepath.Join(sandboxConfig.GetLogDirectory(), config.GetLogPath())
-		if err := c.agentFactory.NewContainerLogger(logPath, agents.Stdout, stdoutPipe).Start(); err != nil {
+		if err = c.agentFactory.NewContainerLogger(logPath, agents.Stdout, stdoutPipe).Start(); err != nil {
 			return fmt.Errorf("failed to start container stdout logger: %v", err)
 		}
 		// Only redirect stderr when there is no tty.
 		if !config.GetTty() {
-			if err := c.agentFactory.NewContainerLogger(logPath, agents.Stderr, stderrPipe).Start(); err != nil {
+			if err = c.agentFactory.NewContainerLogger(logPath, agents.Stderr, stderrPipe).Start(); err != nil {
 				return fmt.Errorf("failed to start container stderr logger: %v", err)
 			}
 		}
@@ -192,7 +192,7 @@ func (c *criContainerdService) startContainer(ctx context.Context, id string, me
 	}
 
 	// Update container start timestamp.
-	meta.Pid = createResp.Pid
+	status.Pid = createResp.Pid
-	meta.StartedAt = time.Now().UnixNano()
+	status.StartedAt = time.Now().UnixNano()
 	return nil
 }

pkg/server/container_start_test.go

@@ -27,27 +27,29 @@ import (
 	"github.com/containerd/containerd/api/types/mount"
 	"github.com/containerd/containerd/api/types/task"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func TestStartContainer(t *testing.T) {
 	testID := "test-id"
 	testSandboxID := "test-sandbox-id"
-	testMetadata := &metadata.ContainerMetadata{
+	testMetadata := containerstore.Metadata{
 		ID:        testID,
 		Name:      "test-name",
 		SandboxID: testSandboxID,
-		CreatedAt: time.Now().UnixNano(),
 	}
-	testSandboxMetadata := &metadata.SandboxMetadata{
+	testStatus := &containerstore.Status{CreatedAt: time.Now().UnixNano()}
+	testSandbox := &sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:   testSandboxID,
 			Name: "test-sandbox-name",
+		},
 	}
 	testSandboxContainer := &task.Task{
 		ID:     testSandboxID,
@@ -56,8 +58,8 @@ func TestStartContainer(t *testing.T) {
 	}
 	testMounts := []*mount.Mount{{Type: "bind", Source: "test-source"}}
 	for desc, test := range map[string]struct {
-		containerMetadata          *metadata.ContainerMetadata
+		status                     *containerstore.Status
-		sandboxMetadata            *metadata.SandboxMetadata
+		sandbox                    *sandboxstore.Sandbox
 		sandboxContainerdContainer *task.Task
 		snapshotMountsErr          bool
 		prepareFIFOErr             error
@@ -67,50 +69,44 @@ func TestStartContainer(t *testing.T) {
 		expectCalls                []string
 		expectErr                  bool
 	}{
-		"should return error when container metadata does not exist": {
+		"should return error when container does not exist": {
-			containerMetadata:          nil,
+			status:                     nil,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			expectCalls:                []string{},
 			expectErr:                  true,
 		},
 		"should return error when container is not in created state": {
-			containerMetadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:        testID,
-				Name:      "test-name",
-				SandboxID: testSandboxID,
 				CreatedAt: time.Now().UnixNano(),
 				StartedAt: time.Now().UnixNano(),
 			},
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			expectCalls:                []string{},
 			expectErr:                  true,
 		},
 		"should return error when container is in removing state": {
-			containerMetadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:        testID,
-				Name:      "test-name",
-				SandboxID: testSandboxID,
 				CreatedAt: time.Now().UnixNano(),
 				Removing:  true,
 			},
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			expectCalls:                []string{},
 			expectErr:                  true,
 		},
 		"should return error when sandbox does not exist": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            nil,
+			sandbox:                    nil,
 			sandboxContainerdContainer: testSandboxContainer,
 			expectStateChange:          true,
 			expectCalls:                []string{},
 			expectErr:                  true,
 		},
 		"should return error when sandbox is not running": {
-			containerMetadata: testMetadata,
+			status:  testStatus,
-			sandboxMetadata:   testSandboxMetadata,
+			sandbox: testSandbox,
 			sandboxContainerdContainer: &task.Task{
 				ID:     testSandboxID,
 				Pid:    uint32(4321),
@@ -121,8 +117,8 @@ func TestStartContainer(t *testing.T) {
 			expectErr:         true,
 		},
 		"should return error when snapshot mounts fails": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			snapshotMountsErr:          true,
 			expectStateChange:          true,
@@ -130,8 +126,8 @@ func TestStartContainer(t *testing.T) {
 			expectErr:                  true,
 		},
 		"should return error when fail to open streaming pipes": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			prepareFIFOErr:             errors.New("open error"),
 			expectStateChange:          true,
@@ -139,8 +135,8 @@ func TestStartContainer(t *testing.T) {
 			expectErr:                  true,
 		},
 		"should return error when fail to create container": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			createContainerErr:         errors.New("create error"),
 			expectStateChange:          true,
@@ -148,8 +144,8 @@ func TestStartContainer(t *testing.T) {
 			expectErr:                  true,
 		},
 		"should return error when fail to start container": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			startContainerErr:          errors.New("start error"),
 			expectStateChange:          true,
@@ -158,8 +154,8 @@ func TestStartContainer(t *testing.T) {
 			expectErr:   true,
 		},
 		"should be able to start container successfully": {
-			containerMetadata:          testMetadata,
+			status:                     testStatus,
-			sandboxMetadata:            testSandboxMetadata,
+			sandbox:                    testSandbox,
 			sandboxContainerdContainer: testSandboxContainer,
 			expectStateChange:          true,
 			expectCalls:                []string{"info", "create", "start"},
@@ -171,11 +167,16 @@ func TestStartContainer(t *testing.T) {
 		fake := c.taskService.(*servertesting.FakeExecutionClient)
 		fakeOS := c.os.(*ostesting.FakeOS)
 		fakeSnapshotClient := WithFakeSnapshotClient(c)
-		if test.containerMetadata != nil {
+		if test.status != nil {
-			assert.NoError(t, c.containerStore.Create(*test.containerMetadata))
+			cntr, err := containerstore.NewContainer(
+				testMetadata,
+				*test.status,
+			)
+			assert.NoError(t, err)
+			assert.NoError(t, c.containerStore.Add(cntr))
 		}
-		if test.sandboxMetadata != nil {
+		if test.sandbox != nil {
-			assert.NoError(t, c.sandboxStore.Create(*test.sandboxMetadata))
+			assert.NoError(t, c.sandboxStore.Add(*test.sandbox))
 		}
 		if test.sandboxContainerdContainer != nil {
 			fake.SetFakeTasks([]task.Task{*test.sandboxContainerdContainer})
@@ -206,35 +207,39 @@ func TestStartContainer(t *testing.T) {
 			assert.NoError(t, err)
 			assert.NotNil(t, resp)
 		}
-		// Check container state.
+		// Skip following validation if no container is injected initially.
-		meta, err := c.containerStore.Get(testID)
+		if test.status == nil {
-		if !test.expectStateChange {
-			// Do not check the error, because container may not exist
-			// in the test case.
-			assert.Equal(t, meta, test.containerMetadata)
 			continue
 		}
+		// Check container state.
+		cntr, err := c.containerStore.Get(testID)
 		assert.NoError(t, err)
-		require.NotNil(t, meta)
+		status := cntr.Status.Get()
+		if !test.expectStateChange {
+			assert.Equal(t, testMetadata, cntr.Metadata)
+			assert.Equal(t, *test.status, status)
+			continue
+		}
 		if test.expectErr {
 			t.Logf("container state should be in exited state when fail to start")
-			assert.Equal(t, runtime.ContainerState_CONTAINER_EXITED, meta.State())
+			assert.Equal(t, runtime.ContainerState_CONTAINER_EXITED, status.State())
-			assert.Zero(t, meta.Pid)
+			assert.Zero(t, status.Pid)
-			assert.EqualValues(t, errorStartExitCode, meta.ExitCode)
+			assert.EqualValues(t, errorStartExitCode, status.ExitCode)
-			assert.Equal(t, errorStartReason, meta.Reason)
+			assert.Equal(t, errorStartReason, status.Reason)
-			assert.NotEmpty(t, meta.Message)
+			assert.NotEmpty(t, status.Message)
 			_, err := fake.Info(context.Background(), &execution.InfoRequest{ContainerID: testID})
 			assert.True(t, isContainerdGRPCNotFoundError(err),
 				"containerd task should be cleaned up when fail to start")
 			continue
 		}
 		t.Logf("container state should be running when start successfully")
-		assert.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, meta.State())
+		assert.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, status.State())
 		info, err := fake.Info(context.Background(), &execution.InfoRequest{ContainerID: testID})
 		assert.NoError(t, err)
 		pid := info.Task.Pid
-		assert.Equal(t, pid, meta.Pid)
+		assert.Equal(t, pid, status.Pid)
 		assert.Equal(t, task.StatusRunning, info.Task.Status)
+		// Check runtime spec
 		calls := fake.GetCalledDetails()
 		createOpts, ok := calls[1].Argument.(*execution.CreateRequest)
 		assert.True(t, ok, "2nd call should be create")

pkg/server/container_status.go

@@ -21,10 +21,9 @@ import (
 
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
-
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 // ContainerStatus inspects the container and returns the status.
@@ -36,22 +35,23 @@ func (c *criContainerdService) ContainerStatus(ctx context.Context, r *runtime.C
 		}
 	}()
 
-	meta, err := c.containerStore.Get(r.GetContainerId())
+	container, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("an error occurred when try to find container %q: %v", r.GetContainerId(), err)
 	}
 
 	return &runtime.ContainerStatusResponse{
-		Status: toCRIContainerStatus(meta),
+		Status: toCRIContainerStatus(container),
 	}, nil
 }
 
-// toCRIContainerStatus converts container metadata to CRI container status.
+// toCRIContainerStatus converts internal container object to CRI container status.
-func toCRIContainerStatus(meta *metadata.ContainerMetadata) *runtime.ContainerStatus {
+func toCRIContainerStatus(container containerstore.Container) *runtime.ContainerStatus {
-	state := meta.State()
+	meta := container.Metadata
-	reason := meta.Reason
+	status := container.Status.Get()
-	if state == runtime.ContainerState_CONTAINER_EXITED && reason == "" {
+	reason := status.Reason
-		if meta.ExitCode == 0 {
+	if status.State() == runtime.ContainerState_CONTAINER_EXITED && reason == "" {
+		if status.ExitCode == 0 {
 			reason = completeExitReason
 		} else {
 			reason = errorExitReason
@@ -60,15 +60,15 @@ func toCRIContainerStatus(meta *metadata.ContainerMetadata) *runtime.ContainerSt
 	return &runtime.ContainerStatus{
 		Id:          meta.ID,
 		Metadata:    meta.Config.GetMetadata(),
-		State:       state,
+		State:       status.State(),
-		CreatedAt:   meta.CreatedAt,
+		CreatedAt:   status.CreatedAt,
-		StartedAt:   meta.StartedAt,
+		StartedAt:   status.StartedAt,
-		FinishedAt:  meta.FinishedAt,
+		FinishedAt:  status.FinishedAt,
-		ExitCode:    meta.ExitCode,
+		ExitCode:    status.ExitCode,
 		Image:       meta.Config.GetImage(),
 		ImageRef:    meta.ImageRef,
 		Reason:      reason,
-		Message:     meta.Message,
+		Message:     status.Message,
 		Labels:      meta.Config.GetLabels(),
 		Annotations: meta.Config.GetAnnotations(),
 		Mounts:      meta.Config.GetMounts(),

pkg/server/container_status_test.go

@@ -22,13 +22,12 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/net/context"
-
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
-func getContainerStatusTestData() (*metadata.ContainerMetadata, *runtime.ContainerStatus) {
+func getContainerStatusTestData() (*containerstore.Metadata, *containerstore.Status, *runtime.ContainerStatus) {
 	testID := "test-id"
 	config := &runtime.ContainerConfig{
 		Metadata: &runtime.ContainerMetadata{
@@ -47,17 +46,18 @@ func getContainerStatusTestData() (*metadata.ContainerMetadata, *runtime.Contain
 	createdAt := time.Now().UnixNano()
 	startedAt := time.Now().UnixNano()
 
-	metadata := &metadata.ContainerMetadata{
+	metadata := &containerstore.Metadata{
 		ID:        testID,
 		Name:      "test-long-name",
 		SandboxID: "test-sandbox-id",
 		Config:    config,
 		ImageRef:  "test-image-ref",
+	}
+	status := &containerstore.Status{
 		Pid:       1234,
 		CreatedAt: createdAt,
 		StartedAt: startedAt,
 	}
-
 	expected := &runtime.ContainerStatus{
 		Id:          testID,
 		Metadata:    config.GetMetadata(),
@@ -72,7 +72,7 @@ func getContainerStatusTestData() (*metadata.ContainerMetadata, *runtime.Contain
 		Mounts:      config.GetMounts(),
 	}
 
-	return metadata, expected
+	return metadata, status, expected
 }
 
 func TestToCRIContainerStatus(t *testing.T) {
@@ -110,19 +110,21 @@ func TestToCRIContainerStatus(t *testing.T) {
 			expectedReason: errorExitReason,
 		},
 	} {
-		meta, expected := getContainerStatusTestData()
+		metadata, status, expected := getContainerStatusTestData()
-		// Update metadata with test case.
+		// Update status with test case.
-		meta.FinishedAt = test.finishedAt
+		status.FinishedAt = test.finishedAt
-		meta.ExitCode = test.exitCode
+		status.ExitCode = test.exitCode
-		meta.Reason = test.reason
+		status.Reason = test.reason
-		meta.Message = test.message
+		status.Message = test.message
+		container, err := containerstore.NewContainer(*metadata, *status)
+		assert.NoError(t, err)
 		// Set expectation based on test case.
 		expected.State = test.expectedState
 		expected.Reason = test.expectedReason
 		expected.FinishedAt = test.finishedAt
 		expected.ExitCode = test.exitCode
 		expected.Message = test.message
-		assert.Equal(t, expected, toCRIContainerStatus(meta), desc)
+		assert.Equal(t, expected, toCRIContainerStatus(container), desc)
 	}
 }
 
@@ -151,14 +153,16 @@ func TestContainerStatus(t *testing.T) {
 	} {
 		t.Logf("TestCase %q", desc)
 		c := newTestCRIContainerdService()
-		meta, expected := getContainerStatusTestData()
+		metadata, status, expected := getContainerStatusTestData()
-		// Update metadata with test case.
+		// Update status with test case.
-		meta.FinishedAt = test.finishedAt
+		status.FinishedAt = test.finishedAt
-		meta.Reason = test.reason
+		status.Reason = test.reason
+		container, err := containerstore.NewContainer(*metadata, *status)
+		assert.NoError(t, err)
 		if test.exist {
-			assert.NoError(t, c.containerStore.Create(*meta))
+			assert.NoError(t, c.containerStore.Add(container))
 		}
-		resp, err := c.ContainerStatus(context.Background(), &runtime.ContainerStatusRequest{ContainerId: meta.ID})
+		resp, err := c.ContainerStatus(context.Background(), &runtime.ContainerStatusRequest{ContainerId: container.ID})
 		if test.expectErr {
 			assert.Error(t, err)
 			assert.Nil(t, resp)

pkg/server/container_stop.go

@@ -27,7 +27,8 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 const (
@@ -50,12 +51,12 @@ func (c *criContainerdService) StopContainer(ctx context.Context, r *runtime.Sto
 	}()
 
 	// Get container config from container store.
-	meta, err := c.containerStore.Get(r.GetContainerId())
+	container, err := c.containerStore.Get(r.GetContainerId())
 	if err != nil {
 		return nil, fmt.Errorf("an error occurred when try to find container %q: %v", r.GetContainerId(), err)
 	}
 
-	if err := c.stopContainer(ctx, meta, time.Duration(r.GetTimeout())*time.Second); err != nil {
+	if err := c.stopContainer(ctx, container, time.Duration(r.GetTimeout())*time.Second); err != nil {
 		return nil, err
 	}
 
@@ -63,32 +64,33 @@ func (c *criContainerdService) StopContainer(ctx context.Context, r *runtime.Sto
 }
 
 // stopContainer stops a container based on the container metadata.
-func (c *criContainerdService) stopContainer(ctx context.Context, meta *metadata.ContainerMetadata, timeout time.Duration) error {
+func (c *criContainerdService) stopContainer(ctx context.Context, container containerstore.Container, timeout time.Duration) error {
-	id := meta.ID
+	id := container.ID
 
 	// Return without error if container is not running. This makes sure that
 	// stop only takes real action after the container is started.
-	if meta.State() != runtime.ContainerState_CONTAINER_RUNNING {
+	state := container.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
 		glog.V(2).Infof("Container to stop %q is not running, current state %q",
-			id, criContainerStateToString(meta.State()))
+			id, criContainerStateToString(state))
 		return nil
 	}
 
 	if timeout > 0 {
 		stopSignal := unix.SIGTERM
-		imageMeta, err := c.imageMetadataStore.Get(meta.ImageRef)
+		image, err := c.imageStore.Get(container.ImageRef)
 		if err != nil {
 			// NOTE(random-liu): It's possible that the container is stopped,
 			// deleted and image is garbage collected before this point. However,
 			// the chance is really slim, even it happens, it's still fine to return
 			// an error here.
-			return fmt.Errorf("failed to get image metadata %q: %v", meta.ImageRef, err)
+			return fmt.Errorf("failed to get image metadata %q: %v", container.ImageRef, err)
 		}
-		if imageMeta.Config.StopSignal != "" {
+		if image.Config.StopSignal != "" {
-			stopSignal, err = signal.ParseSignal(imageMeta.Config.StopSignal)
+			stopSignal, err = signal.ParseSignal(image.Config.StopSignal)
 			if err != nil {
 				return fmt.Errorf("failed to parse stop signal %q: %v",
-					imageMeta.Config.StopSignal, err)
+					image.Config.StopSignal, err)
 			}
 		}
 		glog.V(2).Infof("Stop container %q with signal %v", id, stopSignal)
@@ -140,10 +142,10 @@ func (c *criContainerdService) waitContainerStop(ctx context.Context, id string,
 	defer timeoutTimer.Stop()
 	for {
 		// Poll once before waiting for stopCheckPollInterval.
-		meta, err := c.containerStore.Get(id)
+		container, err := c.containerStore.Get(id)
 		if err != nil {
-			if !metadata.IsNotExistError(err) {
+			if err != store.ErrNotExist {
-				return fmt.Errorf("failed to get container %q metadata: %v", id, err)
+				return fmt.Errorf("failed to get container %q: %v", id, err)
 			}
 			// Do not return error here because container was removed means
 			// it is already stopped.
@@ -151,7 +153,7 @@ func (c *criContainerdService) waitContainerStop(ctx context.Context, id string,
 			return nil
 		}
 		// TODO(random-liu): Use channel with event handler instead of polling.
-		if meta.State() == runtime.ContainerState_CONTAINER_EXITED {
+		if container.Status.Get().State() == runtime.ContainerState_CONTAINER_EXITED {
 			return nil
 		}
 		select {

pkg/server/container_stop_test.go

@@ -29,21 +29,21 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 func TestWaitContainerStop(t *testing.T) {
 	id := "test-id"
 	for desc, test := range map[string]struct {
-		metadata  *metadata.ContainerMetadata
+		status    *containerstore.Status
 		cancel    bool
 		timeout   time.Duration
 		expectErr bool
 	}{
 		"should return error if timeout exceeds": {
-			metadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:        id,
 				CreatedAt: time.Now().UnixNano(),
 				StartedAt: time.Now().UnixNano(),
 			},
@@ -51,8 +51,7 @@ func TestWaitContainerStop(t *testing.T) {
 			expectErr: true,
 		},
 		"should return error if context is cancelled": {
-			metadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:        id,
 				CreatedAt: time.Now().UnixNano(),
 				StartedAt: time.Now().UnixNano(),
 			},
@@ -61,13 +60,12 @@ func TestWaitContainerStop(t *testing.T) {
 			expectErr: true,
 		},
 		"should not return error if container is removed before timeout": {
-			metadata:  nil,
+			status:    nil,
 			timeout:   time.Hour,
 			expectErr: false,
 		},
 		"should not return error if container is stopped before timeout": {
-			metadata: &metadata.ContainerMetadata{
+			status: &containerstore.Status{
-				ID:         id,
 				CreatedAt:  time.Now().UnixNano(),
 				StartedAt:  time.Now().UnixNano(),
 				FinishedAt: time.Now().UnixNano(),
@@ -77,8 +75,13 @@ func TestWaitContainerStop(t *testing.T) {
 		},
 	} {
 		c := newTestCRIContainerdService()
-		if test.metadata != nil {
+		if test.status != nil {
-			assert.NoError(t, c.containerStore.Create(*test.metadata))
+			container, err := containerstore.NewContainer(
+				containerstore.Metadata{ID: id},
+				*test.status,
+			)
+			assert.NoError(t, err)
+			assert.NoError(t, c.containerStore.Add(container))
 		}
 		ctx := context.Background()
 		if test.cancel {
@@ -94,15 +97,14 @@ func TestWaitContainerStop(t *testing.T) {
 func TestStopContainer(t *testing.T) {
 	testID := "test-id"
 	testPid := uint32(1234)
-	testMetadata := metadata.ContainerMetadata{
+	testImageID := "test-image-id"
-		ID:        testID,
+	testStatus := containerstore.Status{
 		Pid:       testPid,
-		ImageRef:  "test-image-id",
 		CreatedAt: time.Now().UnixNano(),
 		StartedAt: time.Now().UnixNano(),
 	}
-	testImageMetadata := metadata.ImageMetadata{
+	testImage := imagestore.Image{
-		ID:     "test-image-id",
+		ID:     testImageID,
 		Config: &imagespec.ImageConfig{},
 	}
 	testContainer := task.Task{
@@ -111,7 +113,7 @@ func TestStopContainer(t *testing.T) {
 		Status: task.StatusRunning,
 	}
 	for desc, test := range map[string]struct {
-		metadata            *metadata.ContainerMetadata
+		status              *containerstore.Status
 		containerdContainer *task.Task
 		stopSignal          string
 		stopErr             error
@@ -120,20 +122,17 @@ func TestStopContainer(t *testing.T) {
 		expectCalls         []servertesting.CalledDetail
 	}{
 		"should return error when container does not exist": {
-			metadata:    nil,
+			status:      nil,
 			expectErr:   true,
 			expectCalls: []servertesting.CalledDetail{},
 		},
 		"should not return error when container is not running": {
-			metadata: &metadata.ContainerMetadata{
+			status:      &containerstore.Status{CreatedAt: time.Now().UnixNano()},
-				ID:        testID,
-				CreatedAt: time.Now().UnixNano(),
-			},
 			expectErr:   false,
 			expectCalls: []servertesting.CalledDetail{},
 		},
 		"should not return error if containerd task does not exist": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			// Since it's hard to inject event during `StopContainer` is running,
 			// we only test the case that first stop returns error, but container
@@ -166,7 +165,7 @@ func TestStopContainer(t *testing.T) {
 			},
 		},
 		"should not return error if containerd task process already finished": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			stopErr:             errors.New("os: process already finished"),
 			expectErr:           false,
@@ -194,7 +193,7 @@ func TestStopContainer(t *testing.T) {
 			},
 		},
 		"should return error if graceful stop returns random error": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			stopErr:             errors.New("random stop error"),
 			expectErr:           true,
@@ -210,7 +209,7 @@ func TestStopContainer(t *testing.T) {
 			},
 		},
 		"should not return error if containerd task is gracefully stopped": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			expectErr:           false,
 			// deleted by the event monitor.
@@ -230,7 +229,7 @@ func TestStopContainer(t *testing.T) {
 			},
 		},
 		"should use stop signal specified in image config if not empty": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			stopSignal:          "SIGHUP",
 			expectErr:           false,
@@ -251,7 +250,7 @@ func TestStopContainer(t *testing.T) {
 			},
 		},
 		"should directly kill container if timeout is 0": {
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainer,
 			noTimeout:           true,
 			expectErr:           false,
@@ -279,16 +278,24 @@ func TestStopContainer(t *testing.T) {
 		defer fake.Stop()
 		c.taskService = fake
 
-		// Inject metadata.
+		// Inject the container.
-		if test.metadata != nil {
+		if test.status != nil {
-			assert.NoError(t, c.containerStore.Create(*test.metadata))
+			cntr, err := containerstore.NewContainer(
+				containerstore.Metadata{
+					ID:       testID,
+					ImageRef: testImageID,
+				},
+				*test.status,
+			)
+			assert.NoError(t, err)
+			assert.NoError(t, c.containerStore.Add(cntr))
 		}
 		// Inject containerd task.
 		if test.containerdContainer != nil {
 			fake.SetFakeTasks([]task.Task{*test.containerdContainer})
 		}
-		testImageMetadata.Config.StopSignal = test.stopSignal
+		testImage.Config.StopSignal = test.stopSignal
-		assert.NoError(t, c.imageMetadataStore.Create(testImageMetadata))
+		c.imageStore.Add(testImage)
 		if test.stopErr != nil {
 			fake.InjectError("kill", test.stopErr)
 		}

pkg/server/events.go

@@ -25,7 +25,7 @@ import (
 	"github.com/jpillora/backoff"
 	"golang.org/x/net/context"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 const (
@@ -87,12 +87,12 @@ func (c *criContainerdService) handleEvent(e *task.Event) {
 	// fine to leave out that case for now.
 	// TODO(random-liu): [P2] Handle containerd-shim exit.
 	case task.Event_EXIT:
-		meta, err := c.containerStore.Get(e.ID)
+		cntr, err := c.containerStore.Get(e.ID)
 		if err != nil {
-			glog.Errorf("Failed to get container %q metadata: %v", e.ID, err)
+			glog.Errorf("Failed to get container %q: %v", e.ID, err)
 			return
 		}
-		if e.Pid != meta.Pid {
+		if e.Pid != cntr.Status.Get().Pid {
 			// Non-init process died, ignore the event.
 			return
 		}
@@ -103,16 +103,16 @@ func (c *criContainerdService) handleEvent(e *task.Event) {
 			glog.Errorf("Failed to delete container %q: %v", e.ID, err)
 			return
 		}
-		err = c.containerStore.Update(e.ID, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
+		err = cntr.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
 			// If FinishedAt has been set (e.g. with start failure), keep as
 			// it is.
-			if meta.FinishedAt != 0 {
+			if status.FinishedAt != 0 {
-				return meta, nil
+				return status, nil
 			}
-			meta.Pid = 0
+			status.Pid = 0
-			meta.FinishedAt = e.ExitedAt.UnixNano()
+			status.FinishedAt = e.ExitedAt.UnixNano()
-			meta.ExitCode = int32(e.ExitStatus)
+			status.ExitCode = int32(e.ExitStatus)
-			return meta, nil
+			return status, nil
 		})
 		if err != nil {
 			glog.Errorf("Failed to update container %q state: %v", e.ID, err)
@@ -120,11 +120,15 @@ func (c *criContainerdService) handleEvent(e *task.Event) {
 			return
 		}
 	case task.Event_OOM:
-		err := c.containerStore.Update(e.ID, func(meta metadata.ContainerMetadata) (metadata.ContainerMetadata, error) {
+		cntr, err := c.containerStore.Get(e.ID)
-			meta.Reason = oomExitReason
+		if err != nil {
-			return meta, nil
+			glog.Errorf("Failed to get container %q: %v", e.ID, err)
+		}
+		err = cntr.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+			status.Reason = oomExitReason
+			return status, nil
 		})
-		if err != nil && !metadata.IsNotExistError(err) {
+		if err != nil {
 			glog.Errorf("Failed to update container %q oom: %v", e.ID, err)
 			return
 		}

pkg/server/events_test.go

@@ -27,8 +27,8 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
 )
 
 func TestHandleEvent(t *testing.T) {
@@ -36,11 +36,13 @@ func TestHandleEvent(t *testing.T) {
 	testPid := uint32(1234)
 	testCreatedAt := time.Now().UnixNano()
 	testStartedAt := time.Now().UnixNano()
-	// Container metadata in running state.
+	testMetadata := containerstore.Metadata{
-	testMetadata := metadata.ContainerMetadata{
 		ID:        testID,
 		Name:      "test-name",
 		SandboxID: "test-sandbox-id",
+	}
+	// Container status in running state.
+	testStatus := containerstore.Status{
 		Pid:       testPid,
 		CreatedAt: testCreatedAt,
 		StartedAt: testStartedAt,
@@ -53,17 +55,14 @@ func TestHandleEvent(t *testing.T) {
 		ExitStatus: 1,
 		ExitedAt:   testExitedAt,
 	}
-	testFinishedMetadata := metadata.ContainerMetadata{
+	testFinishedStatus := containerstore.Status{
-		ID:         testID,
-		Name:       "test-name",
-		SandboxID:  "test-sandbox-id",
 		Pid:        0,
 		CreatedAt:  testCreatedAt,
 		StartedAt:  testStartedAt,
 		FinishedAt: testExitedAt.UnixNano(),
 		ExitCode:   1,
 	}
-	assert.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, testMetadata.State())
+	assert.Equal(t, runtime.ContainerState_CONTAINER_RUNNING, testStatus.State())
 	testContainerdContainer := task.Task{
 		ID:     testID,
 		Pid:    testPid,
@@ -72,12 +71,12 @@ func TestHandleEvent(t *testing.T) {
 
 	for desc, test := range map[string]struct {
 		event               *task.Event
-		metadata            *metadata.ContainerMetadata
+		status              *containerstore.Status
 		containerdContainer *task.Task
 		containerdErr       error
-		expected            *metadata.ContainerMetadata
+		expected            *containerstore.Status
 	}{
-		"should not update state when no corresponding metadata for event": {
+		"should not update state when no corresponding container for event": {
 			event:    &testExitEvent,
 			expected: nil,
 		},
@@ -89,16 +88,16 @@ func TestHandleEvent(t *testing.T) {
 				ExitStatus: 1,
 				ExitedAt:   testExitedAt,
 			},
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainerdContainer,
-			expected:            &testMetadata,
+			expected:            &testStatus,
 		},
 		"should not update state when fail to delete containerd task": {
 			event:               &testExitEvent,
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainerdContainer,
 			containerdErr:       fmt.Errorf("random error"),
-			expected:            &testMetadata,
+			expected:            &testStatus,
 		},
 		"should not update state for irrelevant events": {
 			event: &task.Event{
@@ -106,31 +105,28 @@ func TestHandleEvent(t *testing.T) {
 				Type: task.Event_PAUSED,
 				Pid:  testPid,
 			},
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainerdContainer,
-			expected:            &testMetadata,
+			expected:            &testStatus,
 		},
 		"should update state when containerd task is already deleted": {
 			event:    &testExitEvent,
-			metadata: &testMetadata,
+			status:   &testStatus,
-			expected: &testFinishedMetadata,
+			expected: &testFinishedStatus,
 		},
 		"should update state when delete containerd task successfully": {
 			event:               &testExitEvent,
-			metadata:            &testMetadata,
+			status:              &testStatus,
 			containerdContainer: &testContainerdContainer,
-			expected:            &testFinishedMetadata,
+			expected:            &testFinishedStatus,
 		},
 		"should update exit reason when container is oom killed": {
 			event: &task.Event{
 				ID:   testID,
 				Type: task.Event_OOM,
 			},
-			metadata: &testMetadata,
+			status: &testStatus,
-			expected: &metadata.ContainerMetadata{
+			expected: &containerstore.Status{
-				ID:        testID,
-				Name:      "test-name",
-				SandboxID: "test-sandbox-id",
 				Pid:       testPid,
 				CreatedAt: testCreatedAt,
 				StartedAt: testStartedAt,
@@ -148,10 +144,14 @@ func TestHandleEvent(t *testing.T) {
 		if test.event != nil {
 			fakeEvents.Events <- test.event
 		}
-		// Inject metadata.
+		// Inject internal container object.
-		if test.metadata != nil {
+		if test.status != nil {
-			// Make sure that original data will not be changed.
+			cntr, err := containerstore.NewContainer( // nolint: vetshadow
-			assert.NoError(t, c.containerStore.Create(*test.metadata))
+				testMetadata,
+				*test.status,
+			)
+			assert.NoError(t, err)
+			assert.NoError(t, c.containerStore.Add(cntr))
 		}
 		// Inject containerd task.
 		if test.containerdContainer != nil {
@@ -161,8 +161,12 @@ func TestHandleEvent(t *testing.T) {
 		if test.containerdErr != nil {
 			fake.InjectError("delete", test.containerdErr)
 		}
-		c.handleEventStream(e)
+		assert.NoError(t, c.handleEventStream(e))
-		got, _ := c.containerStore.Get(testID)
+		if test.expected == nil {
-		assert.Equal(t, test.expected, got)
+			continue
+		}
+		got, err := c.containerStore.Get(testID)
+		assert.NoError(t, err)
+		assert.Equal(t, *test.expected, got.Status.Get())
 	}
 }

pkg/server/helpers.go

@@ -36,7 +36,8 @@ import (
 	"google.golang.org/grpc/codes"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 const (
@@ -328,7 +329,7 @@ func getRepoDigestAndTag(namedRef reference.Named, digest imagedigest.Digest, sc
 
 // localResolve resolves image reference locally and returns corresponding image metadata. It returns
 // nil without error if the reference doesn't exist.
-func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*metadata.ImageMetadata, error) {
+func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*imagestore.Image, error) {
 	_, err := imagedigest.Parse(ref)
 	if err != nil {
 		// ref is not image id, try to resolve it locally.
@@ -336,7 +337,7 @@ func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*m
 		if err != nil {
 			return nil, fmt.Errorf("invalid image reference %q: %v", ref, err)
 		}
-		image, err := c.imageStoreService.Get(ctx, normalized.String())
+		imageInContainerd, err := c.imageStoreService.Get(ctx, normalized.String())
 		if err != nil {
 			if containerdmetadata.IsNotFound(err) {
 				return nil, nil
@@ -344,21 +345,21 @@ func (c *criContainerdService) localResolve(ctx context.Context, ref string) (*m
 			return nil, fmt.Errorf("an error occurred when getting image %q from containerd image store: %v",
 				normalized.String(), err)
 		}
-		desc, err := image.Config(ctx, c.contentStoreService)
+		desc, err := imageInContainerd.Config(ctx, c.contentStoreService)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get image config descriptor: %v", err)
 		}
 		ref = desc.Digest.String()
 	}
 	imageID := ref
-	meta, err := c.imageMetadataStore.Get(imageID)
+	image, err := c.imageStore.Get(imageID)
 	if err != nil {
-		if metadata.IsNotExistError(err) {
+		if err == store.ErrNotExist {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("failed to get image %q metadata: %v", imageID, err)
 	}
-	return meta, nil
+	return &image, nil
 }
 
 // getUserFromImage gets uid or user name of the image user.
@@ -382,13 +383,13 @@ func getUserFromImage(user string) (*int64, string) {
 
 // ensureImageExists returns corresponding metadata of the image reference, if image is not
 // pulled yet, the function will pull the image.
-func (c *criContainerdService) ensureImageExists(ctx context.Context, ref string) (*metadata.ImageMetadata, error) {
+func (c *criContainerdService) ensureImageExists(ctx context.Context, ref string) (*imagestore.Image, error) {
-	meta, err := c.localResolve(ctx, ref)
+	image, err := c.localResolve(ctx, ref)
 	if err != nil {
 		return nil, fmt.Errorf("failed to resolve image %q: %v", ref, err)
 	}
-	if meta != nil {
+	if image != nil {
-		return meta, nil
+		return image, nil
 	}
 	// Pull image to ensure the image exists
 	resp, err := c.PullImage(ctx, &runtime.PullImageRequest{Image: &runtime.ImageSpec{Image: ref}})
@@ -396,10 +397,10 @@ func (c *criContainerdService) ensureImageExists(ctx context.Context, ref string
 		return nil, fmt.Errorf("failed to pull image %q: %v", ref, err)
 	}
 	imageID := resp.GetImageRef()
-	meta, err = c.imageMetadataStore.Get(imageID)
+	newImage, err := c.imageStore.Get(imageID)
 	if err != nil {
 		// It's still possible that someone removed the image right after it is pulled.
 		return nil, fmt.Errorf("failed to get image %q metadata after pulling: %v", imageID, err)
 	}
-	return meta, nil
+	return &newImage, nil
 }

pkg/server/image_list.go

@@ -17,13 +17,11 @@ limitations under the License.
 package server
 
 import (
-	"fmt"
-
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 // ListImages lists existing images.
@@ -37,12 +35,10 @@ func (c *criContainerdService) ListImages(ctx context.Context, r *runtime.ListIm
 		}
 	}()
 
-	imageMetadataA, err := c.imageMetadataStore.List()
+	imagesInStore := c.imageStore.List()
-	if err != nil {
+
-		return nil, fmt.Errorf("failed to list image metadata from store: %v", err)
-	}
 	var images []*runtime.Image
-	for _, image := range imageMetadataA {
+	for _, image := range imagesInStore {
 		// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
 		// doesn't exist?
 		images = append(images, toCRIImage(image))
@@ -51,8 +47,8 @@ func (c *criContainerdService) ListImages(ctx context.Context, r *runtime.ListIm
 	return &runtime.ListImagesResponse{Images: images}, nil
 }
 
-// toCRIImage converts image metadata to CRI image type.
+// toCRIImage converts image to CRI image type.
-func toCRIImage(image *metadata.ImageMetadata) *runtime.Image {
+func toCRIImage(image imagestore.Image) *runtime.Image {
 	runtimeImage := &runtime.Image{
 		Id:          image.ID,
 		RepoTags:    image.RepoTags,

pkg/server/image_list_test.go

@@ -25,12 +25,12 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 func TestListImages(t *testing.T) {
 	c := newTestCRIContainerdService()
-	imagesInStore := []metadata.ImageMetadata{
+	imagesInStore := []imagestore.Image{
 		{
 			ID:          "test-id-1",
 			ChainID:     "test-chainid-1",
@@ -87,7 +87,7 @@ func TestListImages(t *testing.T) {
 	}
 
 	for _, i := range imagesInStore {
-		assert.NoError(t, c.imageMetadataStore.Create(i))
+		c.imageStore.Add(i)
 	}
 
 	resp, err := c.ListImages(context.Background(), &runtime.ListImagesRequest{})

pkg/server/image_pull.go

@@ -37,7 +37,7 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 // For image management:
@@ -87,60 +87,41 @@ func (c *criContainerdService) PullImage(ctx context.Context, r *runtime.PullIma
 				r.GetImage().GetImage(), retRes.GetImageRef())
 		}
 	}()
-	image := r.GetImage().GetImage()
+	imageRef := r.GetImage().GetImage()
 
 	// TODO(mikebrow): add truncIndex for image id
-	imageID, repoTag, repoDigest, err := c.pullImage(ctx, image, r.GetAuth())
+	imageID, repoTag, repoDigest, err := c.pullImage(ctx, imageRef, r.GetAuth())
 	if err != nil {
-		return nil, fmt.Errorf("failed to pull image %q: %v", image, err)
+		return nil, fmt.Errorf("failed to pull image %q: %v", imageRef, err)
 	}
-	glog.V(4).Infof("Pulled image %q with image id %q, repo tag %q, repo digest %q", image, imageID,
+	glog.V(4).Infof("Pulled image %q with image id %q, repo tag %q, repo digest %q", imageRef, imageID,
 		repoTag, repoDigest)
 
-	_, err = c.imageMetadataStore.Get(imageID)
-	if err != nil && !metadata.IsNotExistError(err) {
-		return nil, fmt.Errorf("failed to get image %q metadata: %v", imageID, err)
-	}
-	// There is a known race here because the image metadata could be created after `Get`.
-	// TODO(random-liu): [P1] Do not use metadata store. Use simple in-memory data structure to
-	// maintain the id -> information index. And use the container image store as backup and
-	// recover in-memory state during startup.
-	if err == nil {
-		// Update existing image metadata.
-		if err := c.imageMetadataStore.Update(imageID, func(m metadata.ImageMetadata) (metadata.ImageMetadata, error) {
-			updateImageMetadata(&m, repoTag, repoDigest)
-			return m, nil
-		}); err != nil {
-			return nil, fmt.Errorf("failed to update image %q metadata: %v", imageID, err)
-		}
-		return &runtime.PullImageResponse{ImageRef: imageID}, err
-	}
-
 	// Get image information.
-	chainID, size, config, err := c.getImageInfo(ctx, image)
+	chainID, size, config, err := c.getImageInfo(ctx, imageRef)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get image %q information: %v", image, err)
+		return nil, fmt.Errorf("failed to get image %q information: %v", imageRef, err)
 	}
-
+	image := imagestore.Image{
-	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
-	// in-memory image metadata, it's only for in-memory indexing. The image could be removed
-	// by someone else anytime, before/during/after we create the metadata. We should always
-	// check the actual state in containerd before using the image or returning status of the
-	// image.
-
-	// Create corresponding image metadata.
-	newMeta := metadata.ImageMetadata{
 		ID:      imageID,
 		ChainID: chainID.String(),
 		Size:    size,
 		Config:  config,
 	}
-	// Add the image reference used into repo tags. Note if the image is pulled with
+
-	// repo digest, it will also be added in to repo tags, which is fine.
+	if repoDigest != "" {
-	updateImageMetadata(&newMeta, repoTag, repoDigest)
+		image.RepoDigests = []string{repoDigest}
-	if err := c.imageMetadataStore.Create(newMeta); err != nil {
-		return nil, fmt.Errorf("failed to create image %q metadata: %v", imageID, err)
 	}
+	if repoTag != "" {
+		image.RepoTags = []string{repoTag}
+	}
+	c.imageStore.Add(image)
+
+	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
+	// in-memory image store, it's only for in-memory indexing. The image could be removed
+	// by someone else anytime, before/during/after we create the metadata. We should always
+	// check the actual state in containerd before using the image or returning status of the
+	// image.
 	return &runtime.PullImageResponse{ImageRef: imageID}, err
 }
 
@@ -393,29 +374,3 @@ func (c *criContainerdService) waitForResourcesDownloading(ctx context.Context,
 		}
 	}
 }
-
-// insertToStringSlice is a helper function to insert a string into the string slice
-// if the string is not in the slice yet.
-func insertToStringSlice(ss []string, s string) []string {
-	found := false
-	for _, str := range ss {
-		if s == str {
-			found = true
-			break
-		}
-	}
-	if !found {
-		ss = append(ss, s)
-	}
-	return ss
-}
-
-// updateImageMetadata updates existing image meta with new repoTag and repoDigest.
-func updateImageMetadata(meta *metadata.ImageMetadata, repoTag, repoDigest string) {
-	if repoTag != "" {
-		meta.RepoTags = insertToStringSlice(meta.RepoTags, repoTag)
-	}
-	if repoDigest != "" {
-		meta.RepoDigests = insertToStringSlice(meta.RepoDigests, repoDigest)
-	}
-}

pkg/server/image_pull_test.go

@@ -24,59 +24,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
-
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 )
 
-func TestUpdateImageMetadata(t *testing.T) {
-	meta := metadata.ImageMetadata{
-		ID:      "test-id",
-		ChainID: "test-chain-id",
-		Size:    1234,
-	}
-	for desc, test := range map[string]struct {
-		repoTags            []string
-		repoDigests         []string
-		repoTag             string
-		repoDigest          string
-		expectedRepoTags    []string
-		expectedRepoDigests []string
-	}{
-		"Add duplicated repo tag and digest": {
-			repoTags:            []string{"a", "b"},
-			repoDigests:         []string{"c", "d"},
-			repoTag:             "a",
-			repoDigest:          "c",
-			expectedRepoTags:    []string{"a", "b"},
-			expectedRepoDigests: []string{"c", "d"},
-		},
-		"Add new repo tag and digest": {
-			repoTags:            []string{"a", "b"},
-			repoDigests:         []string{"c", "d"},
-			repoTag:             "e",
-			repoDigest:          "f",
-			expectedRepoTags:    []string{"a", "b", "e"},
-			expectedRepoDigests: []string{"c", "d", "f"},
-		},
-		"Add empty repo tag and digest": {
-			repoTags:            []string{"a", "b"},
-			repoDigests:         []string{"c", "d"},
-			repoTag:             "",
-			repoDigest:          "",
-			expectedRepoTags:    []string{"a", "b"},
-			expectedRepoDigests: []string{"c", "d"},
-		},
-	} {
-		t.Logf("TestCase %q", desc)
-		m := meta
-		m.RepoTags = test.repoTags
-		m.RepoDigests = test.repoDigests
-		updateImageMetadata(&m, test.repoTag, test.repoDigest)
-		assert.Equal(t, test.expectedRepoTags, m.RepoTags)
-		assert.Equal(t, test.expectedRepoDigests, m.RepoDigests)
-	}
-}
-
 func TestResources(t *testing.T) {
 	const threads = 10
 	var wg sync.WaitGroup

pkg/server/image_remove.go

@@ -19,13 +19,10 @@ package server
 import (
 	"fmt"
 
+	containerdmetadata "github.com/containerd/containerd/metadata"
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
-
-	containerdmetadata "github.com/containerd/containerd/metadata"
-
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 )
 
 // RemoveImage removes the image.
@@ -41,29 +38,25 @@ func (c *criContainerdService) RemoveImage(ctx context.Context, r *runtime.Remov
 			glog.V(2).Infof("RemoveImage %q returns successfully", r.GetImage().GetImage())
 		}
 	}()
-	meta, err := c.localResolve(ctx, r.GetImage().GetImage())
+	image, err := c.localResolve(ctx, r.GetImage().GetImage())
 	if err != nil {
 		return nil, fmt.Errorf("can not resolve %q locally: %v", r.GetImage().GetImage(), err)
 	}
-	if meta == nil {
+	if image == nil {
 		// return empty without error when image not found.
 		return &runtime.RemoveImageResponse{}, nil
 	}
+
 	// Include all image references, including RepoTag, RepoDigest and id.
-	for _, ref := range append(append(meta.RepoTags, meta.RepoDigests...), meta.ID) {
+	for _, ref := range append(append(image.RepoTags, image.RepoDigests...), image.ID) {
 		// TODO(random-liu): Containerd should schedule a garbage collection immediately,
 		// and we may want to wait for the garbage collection to be over here.
 		err = c.imageStoreService.Delete(ctx, ref)
 		if err == nil || containerdmetadata.IsNotFound(err) {
 			continue
 		}
-		return nil, fmt.Errorf("failed to delete image reference %q for image %q: %v", ref, meta.ID, err)
+		return nil, fmt.Errorf("failed to delete image reference %q for image %q: %v", ref, image.ID, err)
-	}
-	if err = c.imageMetadataStore.Delete(meta.ID); err != nil {
-		if metadata.IsNotExistError(err) {
-			return &runtime.RemoveImageResponse{}, nil
-		}
-		return nil, fmt.Errorf("an error occurred when delete image %q matadata: %v", meta.ID, err)
 	}
+	c.imageStore.Delete(image.ID)
 	return &runtime.RemoveImageResponse{}, nil
 }

pkg/server/image_status.go

@@ -35,28 +35,28 @@ func (c *criContainerdService) ImageStatus(ctx context.Context, r *runtime.Image
 				r.GetImage().GetImage(), retRes.GetImage())
 		}
 	}()
-	meta, err := c.localResolve(ctx, r.GetImage().GetImage())
+	image, err := c.localResolve(ctx, r.GetImage().GetImage())
 	if err != nil {
 		return nil, fmt.Errorf("can not resolve %q locally: %v", r.GetImage().GetImage(), err)
 	}
-	if meta == nil {
+	if image == nil {
 		// return empty without error when image not found.
 		return &runtime.ImageStatusResponse{}, nil
 	}
 	// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
 	// doesn't exist?
 	runtimeImage := &runtime.Image{
-		Id:          meta.ID,
+		Id:          image.ID,
-		RepoTags:    meta.RepoTags,
+		RepoTags:    image.RepoTags,
-		RepoDigests: meta.RepoDigests,
+		RepoDigests: image.RepoDigests,
-		Size_:       uint64(meta.Size),
+		Size_:       uint64(image.Size),
 	}
-	uid, username := getUserFromImage(meta.Config.User)
+	uid, username := getUserFromImage(image.Config.User)
 	if uid != nil {
 		runtimeImage.Uid = &runtime.Int64Value{Value: *uid}
 	}
 	runtimeImage.Username = username
 
-	// TODO(mikebrow): write a ImageMetadata to runtim.Image converter
+	// TODO(mikebrow): write a ImageMetadata to runtime.Image converter
 	return &runtime.ImageStatusResponse{Image: runtimeImage}, nil
 }

pkg/server/image_status_test.go

@@ -25,12 +25,12 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 func TestImageStatus(t *testing.T) {
 	testID := "sha256:d848ce12891bf78792cda4a23c58984033b0c397a55e93a1556202222ecc5ed4"
-	meta := metadata.ImageMetadata{
+	image := imagestore.Image{
 		ID:          testID,
 		ChainID:     "test-chain-id",
 		RepoTags:    []string{"a", "b"},
@@ -57,7 +57,7 @@ func TestImageStatus(t *testing.T) {
 	require.NotNil(t, resp)
 	assert.Nil(t, resp.GetImage())
 
-	assert.NoError(t, c.imageMetadataStore.Create(meta))
+	c.imageStore.Add(image)
 
 	t.Logf("should return correct image status for exist image")
 	resp, err = c.ImageStatus(context.Background(), &runtime.ImageStatusRequest{

pkg/server/sandbox_list.go

@@ -27,7 +27,7 @@ import (
 	"github.com/containerd/containerd/api/types/task"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 // ListPodSandbox returns a list of Sandbox.
@@ -39,11 +39,8 @@ func (c *criContainerdService) ListPodSandbox(ctx context.Context, r *runtime.Li
 		}
 	}()
 
-	// List all sandbox metadata from store.
+	// List all sandboxes from store.
-	sandboxesInStore, err := c.sandboxStore.List()
+	sandboxesInStore := c.sandboxStore.List()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list metadata from sandbox store: %v", err)
-	}
 
 	resp, err := c.taskService.List(ctx, &execution.ListRequest{})
 	if err != nil {
@@ -68,7 +65,7 @@ func (c *criContainerdService) ListPodSandbox(ctx context.Context, r *runtime.Li
 			state = runtime.PodSandboxState_SANDBOX_READY
 		}
 
-		sandboxes = append(sandboxes, toCRISandbox(sandboxInStore, state))
+		sandboxes = append(sandboxes, toCRISandbox(sandboxInStore.Metadata, state))
 	}
 
 	sandboxes = c.filterCRISandboxes(sandboxes, r.GetFilter())
@@ -76,7 +73,7 @@ func (c *criContainerdService) ListPodSandbox(ctx context.Context, r *runtime.Li
 }
 
 // toCRISandbox converts sandbox metadata into CRI pod sandbox.
-func toCRISandbox(meta *metadata.SandboxMetadata, state runtime.PodSandboxState) *runtime.PodSandbox {
+func toCRISandbox(meta sandboxstore.Metadata, state runtime.PodSandboxState) *runtime.PodSandbox {
 	return &runtime.PodSandbox{
 		Id:          meta.ID,
 		Metadata:    meta.Config.GetMetadata(),

pkg/server/sandbox_list_test.go

@@ -27,8 +27,8 @@ import (
 
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func TestToCRISandbox(t *testing.T) {
@@ -43,7 +43,7 @@ func TestToCRISandbox(t *testing.T) {
 		Annotations: map[string]string{"c": "d"},
 	}
 	createdAt := time.Now().UnixNano()
-	meta := &metadata.SandboxMetadata{
+	meta := sandboxstore.Metadata{
 		ID:        "test-id",
 		Name:      "test-name",
 		Config:    config,
@@ -137,22 +137,28 @@ func TestListPodSandbox(t *testing.T) {
 
 	fake := c.taskService.(*servertesting.FakeExecutionClient)
 
-	sandboxesInStore := []metadata.SandboxMetadata{
+	sandboxesInStore := []sandboxstore.Sandbox{
 		{
+			Metadata: sandboxstore.Metadata{
 				ID:     "1",
 				Name:   "name-1",
 				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "name-1"}},
 			},
+		},
 		{
+			Metadata: sandboxstore.Metadata{
 				ID:     "2",
 				Name:   "name-2",
 				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "name-2"}},
 			},
+		},
 		{
+			Metadata: sandboxstore.Metadata{
 				ID:     "3",
 				Name:   "name-3",
 				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "name-3"}},
 			},
+		},
 	}
 	sandboxesInContainerd := []task.Task{
 		// Running container with corresponding metadata
@@ -194,7 +200,7 @@ func TestListPodSandbox(t *testing.T) {
 
 	// Inject test metadata
 	for _, s := range sandboxesInStore {
-		c.sandboxStore.Create(s)
+		assert.NoError(t, c.sandboxStore.Add(s))
 	}
 
 	// Inject fake containerd tasks

pkg/server/sandbox_remove.go

@@ -26,7 +26,7 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
 )
 
 // RemovePodSandbox removes the sandbox. If there are running containers in the
@@ -41,7 +41,7 @@ func (c *criContainerdService) RemovePodSandbox(ctx context.Context, r *runtime.
 
 	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
 	if err != nil {
-		if !metadata.IsNotExistError(err) {
+		if err != store.ErrNotExist {
 			return nil, fmt.Errorf("an error occurred when try to find sandbox %q: %v",
 				r.GetPodSandboxId(), err)
 		}
@@ -76,10 +76,7 @@ func (c *criContainerdService) RemovePodSandbox(ctx context.Context, r *runtime.
 	// not rely on this behavior.
 	// TODO(random-liu): Introduce an intermediate state to avoid container creation after
 	// this point.
-	cntrs, err := c.containerStore.List()
+	cntrs := c.containerStore.List()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list all containers: %v", err)
-	}
 	for _, cntr := range cntrs {
 		if cntr.SandboxID != id {
 			continue
@@ -107,15 +104,12 @@ func (c *criContainerdService) RemovePodSandbox(ctx context.Context, r *runtime.
 		glog.V(5).Infof("Remove called for sandbox container %q that does not exist", id, err)
 	}
 
-	// Remove sandbox metadata from metadata store. Note that once the sandbox
+	// Remove sandbox from sandbox store. Note that once the sandbox is successfully
-	// metadata is successfully deleted:
+	// deleted:
 	// 1) ListPodSandbox will not include this sandbox.
 	// 2) PodSandboxStatus and StopPodSandbox will return error.
-	// 3) On-going operations which have held the metadata reference will not be
+	// 3) On-going operations which have held the reference will not be affected.
-	// affected.
+	c.sandboxStore.Delete(id)
-	if err := c.sandboxStore.Delete(id); err != nil {
-		return nil, fmt.Errorf("failed to delete sandbox metadata for %q: %v", id, err)
-	}
 
 	// Release the sandbox name reserved for the sandbox.
 	c.sandboxNameIndex.ReleaseByKey(id)

pkg/server/sandbox_remove_test.go

@@ -27,21 +27,25 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	"github.com/kubernetes-incubator/cri-containerd/pkg/store"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func TestRemovePodSandbox(t *testing.T) {
 	testID := "test-id"
 	testName := "test-name"
-	testMetadata := metadata.SandboxMetadata{
+	testSandbox := sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:   testID,
 			Name: testName,
+		},
 	}
 	for desc, test := range map[string]struct {
 		sandboxTasks       []task.Task
-		injectMetadata     bool
+		injectSandbox      bool
 		removeSnapshotErr  error
 		deleteContainerErr error
 		taskInfoErr        error
@@ -51,58 +55,58 @@ func TestRemovePodSandbox(t *testing.T) {
 		expectCalls        []string
 	}{
 		"should not return error if sandbox does not exist": {
-			injectMetadata:     false,
+			injectSandbox:      false,
 			removeSnapshotErr:  servertesting.SnapshotNotExistError,
 			deleteContainerErr: servertesting.ContainerNotExistError,
 			expectErr:          false,
 			expectCalls:        []string{},
 		},
 		"should not return error if snapshot does not exist": {
-			injectMetadata:    true,
+			injectSandbox:     true,
 			removeSnapshotErr: servertesting.SnapshotNotExistError,
 			expectRemoved:     getSandboxRootDir(testRootDir, testID),
 			expectCalls:       []string{"info"},
 		},
 		"should return error if remove snapshot fails": {
-			injectMetadata:    true,
+			injectSandbox:     true,
 			removeSnapshotErr: fmt.Errorf("arbitrary error"),
 			expectErr:         true,
 			expectCalls:       []string{"info"},
 		},
 		"should return error when sandbox container task is not deleted": {
-			injectMetadata: true,
+			injectSandbox: true,
 			sandboxTasks:  []task.Task{{ID: testID}},
 			expectErr:     true,
 			expectCalls:   []string{"info"},
 		},
 		"should return error when arbitrary containerd error is injected": {
-			injectMetadata: true,
+			injectSandbox: true,
 			taskInfoErr:   fmt.Errorf("arbitrary error"),
 			expectErr:     true,
 			expectCalls:   []string{"info"},
 		},
 		"should return error when error fs error is injected": {
-			injectMetadata: true,
+			injectSandbox: true,
 			injectFSErr:   fmt.Errorf("fs error"),
 			expectRemoved: getSandboxRootDir(testRootDir, testID),
 			expectErr:     true,
 			expectCalls:   []string{"info"},
 		},
 		"should not return error if sandbox container does not exist": {
-			injectMetadata:     true,
+			injectSandbox:      true,
 			deleteContainerErr: servertesting.ContainerNotExistError,
 			expectRemoved:      getSandboxRootDir(testRootDir, testID),
 			expectCalls:        []string{"info"},
 		},
 		"should return error if delete sandbox container fails": {
-			injectMetadata:     true,
+			injectSandbox:      true,
 			deleteContainerErr: fmt.Errorf("arbitrary error"),
 			expectRemoved:      getSandboxRootDir(testRootDir, testID),
 			expectErr:          true,
 			expectCalls:        []string{"info"},
 		},
 		"should be able to successfully delete": {
-			injectMetadata: true,
+			injectSandbox: true,
 			expectRemoved: getSandboxRootDir(testRootDir, testID),
 			expectCalls:   []string{"info"},
 		},
@@ -114,9 +118,9 @@ func TestRemovePodSandbox(t *testing.T) {
 		fakeExecutionClient := c.taskService.(*servertesting.FakeExecutionClient)
 		fakeSnapshotClient := WithFakeSnapshotClient(c)
 		fakeExecutionClient.SetFakeTasks(test.sandboxTasks)
-		if test.injectMetadata {
+		if test.injectSandbox {
 			c.sandboxNameIndex.Reserve(testName, testID)
-			c.sandboxStore.Create(testMetadata)
+			assert.NoError(t, c.sandboxStore.Add(testSandbox))
 		}
 		if test.removeSnapshotErr == nil {
 			fakeSnapshotClient.SetFakeMounts(testID, []*mount.Mount{
@@ -157,10 +161,8 @@ func TestRemovePodSandbox(t *testing.T) {
 		assert.NotNil(t, res)
 		assert.NoError(t, c.sandboxNameIndex.Reserve(testName, testID),
 			"sandbox name should be released")
-		meta, err := c.sandboxStore.Get(testID)
+		_, err = c.sandboxStore.Get(testID)
-		assert.Error(t, err)
+		assert.Equal(t, store.ErrNotExist, err, "sandbox should be removed")
-		assert.True(t, metadata.IsNotExistError(err))
-		assert.Nil(t, meta, "sandbox metadata should be removed")
 		mountsResp, err := fakeSnapshotClient.Mounts(context.Background(), &snapshotapi.MountsRequest{Key: testID})
 		assert.Equal(t, servertesting.SnapshotNotExistError, err, "snapshot should be removed")
 		assert.Nil(t, mountsResp)
@@ -178,38 +180,49 @@ func TestRemovePodSandbox(t *testing.T) {
 func TestRemoveContainersInSandbox(t *testing.T) {
 	testID := "test-id"
 	testName := "test-name"
-	testMetadata := metadata.SandboxMetadata{
+	testSandbox := sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:   testID,
 			Name: testName,
+		},
 	}
-	testContainersMetadata := []*metadata.ContainerMetadata{
+	testContainers := []containerForTest{
 		{
+			metadata: containerstore.Metadata{
 				ID:        "test-cid-1",
 				Name:      "test-cname-1",
 				SandboxID: testID,
-			FinishedAt: time.Now().UnixNano(),
+			},
+			status: containerstore.Status{FinishedAt: time.Now().UnixNano()},
 		},
 		{
+			metadata: containerstore.Metadata{
+
 				ID:        "test-cid-2",
 				Name:      "test-cname-2",
 				SandboxID: testID,
-			FinishedAt: time.Now().UnixNano(),
+			},
+			status: containerstore.Status{FinishedAt: time.Now().UnixNano()},
 		},
 		{
+			metadata: containerstore.Metadata{
 				ID:        "test-cid-3",
 				Name:      "test-cname-3",
 				SandboxID: "other-sandbox-id",
-			FinishedAt: time.Now().UnixNano(),
+			},
+			status: containerstore.Status{FinishedAt: time.Now().UnixNano()},
 		},
 	}
 
 	c := newTestCRIContainerdService()
 	WithFakeSnapshotClient(c)
 	assert.NoError(t, c.sandboxNameIndex.Reserve(testName, testID))
-	assert.NoError(t, c.sandboxStore.Create(testMetadata))
+	assert.NoError(t, c.sandboxStore.Add(testSandbox))
-	for _, cntr := range testContainersMetadata {
+	for _, tc := range testContainers {
-		assert.NoError(t, c.containerNameIndex.Reserve(cntr.Name, cntr.ID))
+		assert.NoError(t, c.containerNameIndex.Reserve(tc.metadata.Name, tc.metadata.ID))
-		assert.NoError(t, c.containerStore.Create(*cntr))
+		cntr, err := tc.toContainer()
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(cntr))
 	}
 
 	res, err := c.RemovePodSandbox(context.Background(), &runtime.RemovePodSandboxRequest{
@@ -218,12 +231,11 @@ func TestRemoveContainersInSandbox(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, res)
 
-	meta, err := c.sandboxStore.Get(testID)
+	_, err = c.sandboxStore.Get(testID)
-	assert.Error(t, err)
+	assert.Equal(t, store.ErrNotExist, err, "sandbox metadata should be removed")
-	assert.True(t, metadata.IsNotExistError(err))
-	assert.Nil(t, meta, "sandbox metadata should be removed")
 
-	cntrs, err := c.containerStore.List()
+	cntrs := c.containerStore.List()
-	assert.NoError(t, err)
+	assert.Len(t, cntrs, 1)
-	assert.Equal(t, testContainersMetadata[2:], cntrs, "container metadata should be removed")
+	assert.Equal(t, testContainers[2].metadata, cntrs[0].Metadata, "container should be removed")
+	assert.Equal(t, testContainers[2].status, cntrs[0].Status.Get(), "container should be removed")
 }

pkg/server/sandbox_run.go

@@ -35,7 +35,7 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
@@ -65,22 +65,23 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		}
 	}()
 
-	// Create initial sandbox metadata.
+	// Create initial internal sandbox object.
-	meta := metadata.SandboxMetadata{
+	sandbox := sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:     id,
 			Name:   name,
 			Config: config,
+		},
 	}
 
 	// Ensure sandbox container image snapshot.
-	imageMeta, err := c.ensureImageExists(ctx, c.sandboxImage)
+	image, err := c.ensureImageExists(ctx, c.sandboxImage)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get sandbox image %q: %v", defaultSandboxImage, err)
 	}
-
+	rootfsMounts, err := c.snapshotService.View(ctx, id, image.ChainID)
-	rootfsMounts, err := c.snapshotService.View(ctx, id, imageMeta.ChainID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to prepare sandbox rootfs %q: %v", imageMeta.ChainID, err)
+		return nil, fmt.Errorf("failed to prepare sandbox rootfs %q: %v", image.ChainID, err)
 	}
 	defer func() {
 		if retErr != nil {
@@ -99,7 +100,7 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	}
 
 	// Create sandbox container.
-	spec, err := c.generateSandboxContainerSpec(id, config, imageMeta.Config)
+	spec, err := c.generateSandboxContainerSpec(id, config, image.Config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate sandbox container spec: %v", err)
 	}
@@ -112,7 +113,7 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		Container: containers.Container{
 			ID: id,
 			// TODO(random-liu): Checkpoint metadata into container labels.
-			Image:   imageMeta.ID,
+			Image:   image.ID,
 			Runtime: defaultRuntime,
 			Spec: &prototypes.Any{
 				TypeUrl: runtimespec.Version,
@@ -205,19 +206,19 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 		}
 	}()
 
-	meta.Pid = createResp.Pid
+	sandbox.Pid = createResp.Pid
-	meta.NetNS = getNetworkNamespace(createResp.Pid)
+	sandbox.NetNS = getNetworkNamespace(createResp.Pid)
 	if !config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostNetwork() {
 		// Setup network for sandbox.
 		// TODO(random-liu): [P2] Replace with permanent network namespace.
 		podName := config.GetMetadata().GetName()
-		if err = c.netPlugin.SetUpPod(meta.NetNS, config.GetMetadata().GetNamespace(), podName, id); err != nil {
+		if err = c.netPlugin.SetUpPod(sandbox.NetNS, config.GetMetadata().GetNamespace(), podName, id); err != nil {
 			return nil, fmt.Errorf("failed to setup network for sandbox %q: %v", id, err)
 		}
 		defer func() {
 			if retErr != nil {
 				// Teardown network if an error is returned.
-				if err := c.netPlugin.TearDownPod(meta.NetNS, config.GetMetadata().GetNamespace(), podName, id); err != nil {
+				if err := c.netPlugin.TearDownPod(sandbox.NetNS, config.GetMetadata().GetNamespace(), podName, id); err != nil {
 					glog.Errorf("failed to destroy network for sandbox %q: %v", id, err)
 				}
 			}
@@ -231,10 +232,9 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	}
 
 	// Add sandbox into sandbox store.
-	meta.CreatedAt = time.Now().UnixNano()
+	sandbox.CreatedAt = time.Now().UnixNano()
-	if err := c.sandboxStore.Create(meta); err != nil {
+	if err := c.sandboxStore.Add(sandbox); err != nil {
-		return nil, fmt.Errorf("failed to add sandbox metadata %+v into store: %v",
+		return nil, fmt.Errorf("failed to add sandbox %+v into store: %v", sandbox, err)
-			meta, err)
 	}
 
 	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil

pkg/server/sandbox_run_test.go

@@ -33,9 +33,9 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
 )
 
 func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
@@ -290,13 +290,13 @@ func TestRunPodSandbox(t *testing.T) {
 		return nopReadWriteCloser{}, nil
 	}
 	testChainID := "test-sandbox-chain-id"
-	imageMetadata := metadata.ImageMetadata{
+	image := imagestore.Image{
 		ID:      testSandboxImage,
 		ChainID: testChainID,
 		Config:  imageConfig,
 	}
-	// Insert sandbox image metadata.
+	// Insert sandbox image.
-	assert.NoError(t, c.imageMetadataStore.Create(imageMetadata))
+	c.imageStore.Add(image)
 	expectContainersClientCalls := []string{"create"}
 	expectSnapshotClientCalls := []string{"view"}
 	expectExecutionClientCalls := []string{"create", "start"}
@@ -349,25 +349,25 @@ func TestRunPodSandbox(t *testing.T) {
 	startID := calls[1].Argument.(*execution.StartRequest).ContainerID
 	assert.Equal(t, id, startID, "start id should be correct")
 
-	meta, err := c.sandboxStore.Get(id)
+	sandbox, err := c.sandboxStore.Get(id)
 	assert.NoError(t, err)
-	assert.Equal(t, id, meta.ID, "metadata id should be correct")
+	assert.Equal(t, id, sandbox.ID, "sandbox id should be correct")
-	err = c.sandboxNameIndex.Reserve(meta.Name, "random-id")
+	err = c.sandboxNameIndex.Reserve(sandbox.Name, "random-id")
-	assert.Error(t, err, "metadata name should be reserved")
+	assert.Error(t, err, "sandbox name should be reserved")
-	assert.Equal(t, config, meta.Config, "metadata config should be correct")
+	assert.Equal(t, config, sandbox.Config, "sandbox config should be correct")
 	// TODO(random-liu): [P2] Add clock interface and use fake clock.
-	assert.NotZero(t, meta.CreatedAt, "metadata CreatedAt should be set")
+	assert.NotZero(t, sandbox.CreatedAt, "sandbox CreatedAt should be set")
 	info, err := fakeExecutionClient.Info(context.Background(), &execution.InfoRequest{ContainerID: id})
 	assert.NoError(t, err)
 	pid := info.Task.Pid
-	assert.Equal(t, meta.NetNS, getNetworkNamespace(pid), "metadata network namespace should be correct")
+	assert.Equal(t, sandbox.NetNS, getNetworkNamespace(pid), "sandbox network namespace should be correct")
 
 	expectedCNICalls := []string{"SetUpPod"}
 	assert.Equal(t, expectedCNICalls, fakeCNIPlugin.GetCalledNames(), "expect SetUpPod should be called")
 	calls = fakeCNIPlugin.GetCalledDetails()
 	pluginArgument := calls[0].Argument.(servertesting.CNIPluginArgument)
 	expectedPluginArgument := servertesting.CNIPluginArgument{
-		NetnsPath:   meta.NetNS,
+		NetnsPath:   sandbox.NetNS,
 		Namespace:   config.GetMetadata().GetNamespace(),
 		Name:        config.GetMetadata().GetName(),
 		ContainerID: id,

pkg/server/sandbox_status.go

@@ -27,7 +27,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 // PodSandboxStatus returns the status of the PodSandbox.
@@ -66,11 +66,11 @@ func (c *criContainerdService) PodSandboxStatus(ctx context.Context, r *runtime.
 		glog.V(4).Infof("GetContainerNetworkStatus returns error: %v", err)
 	}
 
-	return &runtime.PodSandboxStatusResponse{Status: toCRISandboxStatus(sandbox, state, ip)}, nil
+	return &runtime.PodSandboxStatusResponse{Status: toCRISandboxStatus(sandbox.Metadata, state, ip)}, nil
 }
 
 // toCRISandboxStatus converts sandbox metadata into CRI pod sandbox status.
-func toCRISandboxStatus(meta *metadata.SandboxMetadata, state runtime.PodSandboxState, ip string) *runtime.PodSandboxStatus {
+func toCRISandboxStatus(meta sandboxstore.Metadata, state runtime.PodSandboxState, ip string) *runtime.PodSandboxStatus {
 	nsOpts := meta.Config.GetLinux().GetSecurityContext().GetNamespaceOptions()
 	return &runtime.PodSandboxStatus{
 		Id:        meta.ID,

pkg/server/sandbox_status_test.go

@@ -21,16 +21,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containerd/containerd/api/types/task"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
-
-	"github.com/containerd/containerd/api/types/task"
-
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 // Variables used in the following test.
@@ -41,7 +39,7 @@ const (
 	sandboxStatusTestNetNS = "test-netns"
 )
 
-func getSandboxStatusTestData() (*metadata.SandboxMetadata, *runtime.PodSandboxStatus) {
+func getSandboxStatusTestData() (*sandboxstore.Sandbox, *runtime.PodSandboxStatus) {
 	config := &runtime.PodSandboxConfig{
 		Metadata: &runtime.PodSandboxMetadata{
 			Name:      "test-name",
@@ -64,12 +62,14 @@ func getSandboxStatusTestData() (*metadata.SandboxMetadata, *runtime.PodSandboxS
 
 	createdAt := time.Now().UnixNano()
 
-	metadata := &metadata.SandboxMetadata{
+	sandbox := &sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:        sandboxStatusTestID,
 			Name:      "test-name",
 			Config:    config,
 			CreatedAt: createdAt,
 			NetNS:     sandboxStatusTestNetNS,
+		},
 	}
 
 	expectedStatus := &runtime.PodSandboxStatus{
@@ -90,13 +90,13 @@ func getSandboxStatusTestData() (*metadata.SandboxMetadata, *runtime.PodSandboxS
 		Annotations: config.GetAnnotations(),
 	}
 
-	return metadata, expectedStatus
+	return sandbox, expectedStatus
 }
 
 func TestPodSandboxStatus(t *testing.T) {
 	for desc, test := range map[string]struct {
 		sandboxTasks     []task.Task
-		injectMetadata   bool
+		injectSandbox    bool
 		injectErr        error
 		injectIP         bool
 		injectCNIErr     error
@@ -106,7 +106,7 @@ func TestPodSandboxStatus(t *testing.T) {
 		expectedCNICalls []string
 	}{
 		"sandbox status without metadata": {
-			injectMetadata:   false,
+			injectSandbox:    false,
 			expectErr:        true,
 			expectCalls:      []string{},
 			expectedCNICalls: []string{},
@@ -117,7 +117,7 @@ func TestPodSandboxStatus(t *testing.T) {
 				Pid:    1,
 				Status: task.StatusRunning,
 			}},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_READY,
 			expectCalls:      []string{"info"},
 			expectedCNICalls: []string{"GetContainerNetworkStatus"},
@@ -128,14 +128,14 @@ func TestPodSandboxStatus(t *testing.T) {
 				Pid:    1,
 				Status: task.StatusStopped,
 			}},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_NOTREADY,
 			expectCalls:      []string{"info"},
 			expectedCNICalls: []string{"GetContainerNetworkStatus"},
 		},
 		"sandbox status with non-existing sandbox container": {
 			sandboxTasks:     []task.Task{},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_NOTREADY,
 			expectCalls:      []string{"info"},
 			expectedCNICalls: []string{"GetContainerNetworkStatus"},
@@ -146,7 +146,7 @@ func TestPodSandboxStatus(t *testing.T) {
 				Pid:    1,
 				Status: task.StatusRunning,
 			}},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_READY,
 			injectErr:        errors.New("arbitrary error"),
 			expectErr:        true,
@@ -159,7 +159,7 @@ func TestPodSandboxStatus(t *testing.T) {
 				Pid:    1,
 				Status: task.StatusRunning,
 			}},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_READY,
 			expectCalls:      []string{"info"},
 			injectIP:         true,
@@ -171,7 +171,7 @@ func TestPodSandboxStatus(t *testing.T) {
 				Pid:    1,
 				Status: task.StatusRunning,
 			}},
-			injectMetadata:   true,
+			injectSandbox:    true,
 			expectState:      runtime.PodSandboxState_SANDBOX_READY,
 			expectCalls:      []string{"info"},
 			expectedCNICalls: []string{"GetContainerNetworkStatus"},
@@ -179,14 +179,14 @@ func TestPodSandboxStatus(t *testing.T) {
 		},
 	} {
 		t.Logf("TestCase %q", desc)
-		metadata, expect := getSandboxStatusTestData()
+		sandbox, expect := getSandboxStatusTestData()
 		expect.Network.Ip = ""
 		c := newTestCRIContainerdService()
 		fake := c.taskService.(*servertesting.FakeExecutionClient)
 		fakeCNIPlugin := c.netPlugin.(*servertesting.FakeCNIPlugin)
 		fake.SetFakeTasks(test.sandboxTasks)
-		if test.injectMetadata {
+		if test.injectSandbox {
-			assert.NoError(t, c.sandboxStore.Create(*metadata))
+			assert.NoError(t, c.sandboxStore.Add(*sandbox))
 		}
 		if test.injectErr != nil {
 			fake.InjectError("info", test.injectErr)
@@ -195,8 +195,8 @@ func TestPodSandboxStatus(t *testing.T) {
 			fakeCNIPlugin.InjectError("GetContainerNetworkStatus", test.injectCNIErr)
 		}
 		if test.injectIP {
-			fakeCNIPlugin.SetFakePodNetwork(metadata.NetNS, metadata.Config.GetMetadata().GetNamespace(),
+			fakeCNIPlugin.SetFakePodNetwork(sandbox.NetNS, sandbox.Config.GetMetadata().GetNamespace(),
-				metadata.Config.GetMetadata().GetName(), sandboxStatusTestID, sandboxStatusTestIP)
+				sandbox.Config.GetMetadata().GetName(), sandboxStatusTestID, sandboxStatusTestIP)
 			expect.Network.Ip = sandboxStatusTestIP
 		}
 		res, err := c.PodSandboxStatus(context.Background(), &runtime.PodSandboxStatusRequest{

pkg/server/sandbox_stop.go

@@ -50,10 +50,7 @@ func (c *criContainerdService) StopPodSandbox(ctx context.Context, r *runtime.St
 	// and container may still be so production should not rely on this behavior.
 	// TODO(random-liu): Delete the sandbox container before this after permanent network namespace
 	// is introduced, so that no container will be started after that.
-	containers, err := c.containerStore.List()
+	containers := c.containerStore.List()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list all containers: %v", err)
-	}
 	for _, container := range containers {
 		if container.SandboxID != id {
 			continue

pkg/server/sandbox_stop_test.go

@@ -30,14 +30,16 @@ import (
 	"google.golang.org/grpc/codes"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 func TestStopPodSandbox(t *testing.T) {
 	testID := "test-id"
-	testSandbox := metadata.SandboxMetadata{
+	testSandbox := sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:   testID,
 			Name: "test-name",
 			Config: &runtime.PodSandboxConfig{
@@ -47,6 +49,7 @@ func TestStopPodSandbox(t *testing.T) {
 					Namespace: "test-ns",
 				}},
 			NetNS: "test-netns",
+		},
 	}
 	testContainer := task.Task{
 		ID:     testID,
@@ -136,7 +139,7 @@ func TestStopPodSandbox(t *testing.T) {
 		fake.SetFakeTasks(test.sandboxTasks)
 
 		if test.injectSandbox {
-			assert.NoError(t, c.sandboxStore.Create(testSandbox))
+			assert.NoError(t, c.sandboxStore.Add(testSandbox))
 		}
 		if test.injectErr != nil {
 			fake.InjectError("delete", test.injectErr)
@@ -170,7 +173,8 @@ func TestStopPodSandbox(t *testing.T) {
 
 func TestStopContainersInSandbox(t *testing.T) {
 	testID := "test-id"
-	testSandbox := metadata.SandboxMetadata{
+	testSandbox := sandboxstore.Sandbox{
+		Metadata: sandboxstore.Metadata{
 			ID:   testID,
 			Name: "test-name",
 			Config: &runtime.PodSandboxConfig{
@@ -180,29 +184,43 @@ func TestStopContainersInSandbox(t *testing.T) {
 					Namespace: "test-ns",
 				}},
 			NetNS: "test-netns",
+		},
 	}
-	testContainers := []metadata.ContainerMetadata{
+	testContainers := []containerForTest{
 		{
+			metadata: containerstore.Metadata{
 				ID:        "test-cid-1",
 				Name:      "test-cname-1",
 				SandboxID: testID,
+			},
+			status: containerstore.Status{
 				Pid:       2,
 				StartedAt: time.Now().UnixNano(),
 			},
+		},
 		{
+
+			metadata: containerstore.Metadata{
 				ID:        "test-cid-2",
 				Name:      "test-cname-2",
 				SandboxID: testID,
+			},
+			status: containerstore.Status{
 				Pid:       3,
 				StartedAt: time.Now().UnixNano(),
 			},
+		},
 		{
+			metadata: containerstore.Metadata{
 				ID:        "test-cid-3",
 				Name:      "test-cname-3",
 				SandboxID: "other-sandbox-id",
+			},
+			status: containerstore.Status{
 				Pid:       4,
 				StartedAt: time.Now().UnixNano(),
 			},
+		},
 	}
 	testContainerdContainers := []task.Task{
 		{
@@ -232,9 +250,11 @@ func TestStopContainersInSandbox(t *testing.T) {
 	defer fake.Stop()
 	c.taskService = fake
 	fake.SetFakeTasks(testContainerdContainers)
-	assert.NoError(t, c.sandboxStore.Create(testSandbox))
+	c.sandboxStore.Add(testSandbox)
-	for _, cntr := range testContainers {
+	for _, tc := range testContainers {
-		assert.NoError(t, c.containerStore.Create(cntr))
+		cntr, err := tc.toContainer()
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(cntr))
 	}
 
 	fakeCNIPlugin := c.netPlugin.(*servertesting.FakeCNIPlugin)
@@ -257,8 +277,7 @@ func TestStopContainersInSandbox(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, res)
 
-	cntrs, err := c.containerStore.List()
+	cntrs := c.containerStore.List()
-	assert.NoError(t, err)
 	assert.Len(t, cntrs, 3)
 	expectedStates := map[string]runtime.ContainerState{
 		"test-cid-1": runtime.ContainerState_CONTAINER_EXITED,
@@ -268,6 +287,6 @@ func TestStopContainersInSandbox(t *testing.T) {
 	for id, expected := range expectedStates {
 		cntr, err := c.containerStore.Get(id)
 		assert.NoError(t, err)
-		assert.Equal(t, expected, cntr.State())
+		assert.Equal(t, expected, cntr.Status.Get().State())
 	}
 }

pkg/server/service.go

@@ -31,11 +31,12 @@ import (
 	healthapi "google.golang.org/grpc/health/grpc_health_v1"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata/store"
 	osinterface "github.com/kubernetes-incubator/cri-containerd/pkg/os"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/registrar"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/server/agents"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 // k8sContainerdNamespace is the namespace we use to connect containerd.
@@ -57,19 +58,19 @@ type criContainerdService struct {
 	// sandboxImage is the image to use for sandbox container.
 	// TODO(random-liu): Make this configurable via flag.
 	sandboxImage string
-	// sandboxStore stores all sandbox metadata.
+	// sandboxStore stores all resources associated with sandboxes.
-	sandboxStore metadata.SandboxStore
+	sandboxStore *sandboxstore.Store
-	// imageMetadataStore stores all image metadata.
-	imageMetadataStore metadata.ImageMetadataStore
 	// sandboxNameIndex stores all sandbox names and make sure each name
 	// is unique.
 	sandboxNameIndex *registrar.Registrar
-	// containerStore stores all container metadata.
+	// containerStore stores all resources associated with containers.
-	containerStore metadata.ContainerStore
+	containerStore *containerstore.Store
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
-	// containerService is containerd tasks client.
+	// imageStore stores all resources associated with images.
+	imageStore *imagestore.Store
+	// containerService is containerd containers client.
 	containerService containers.ContainersClient
 	// taskService is containerd tasks client.
 	taskService execution.TasksClient
@@ -96,7 +97,7 @@ type criContainerdService struct {
 
 // NewCRIContainerdService returns a new instance of CRIContainerdService
 func NewCRIContainerdService(containerdEndpoint, rootDir, networkPluginBinDir, networkPluginConfDir string) (CRIContainerdService, error) {
-	// TODO(random-liu): [P2] Recover from runtime state and metadata store.
+	// TODO(random-liu): [P2] Recover from runtime state and checkpoint.
 
 	client, err := containerd.New(containerdEndpoint, containerd.WithDefaultNamespace(k8sContainerdNamespace))
 	if err != nil {
@@ -107,9 +108,9 @@ func NewCRIContainerdService(containerdEndpoint, rootDir, networkPluginBinDir, n
 		os:                  osinterface.RealOS{},
 		rootDir:             rootDir,
 		sandboxImage:        defaultSandboxImage,
-		sandboxStore:        metadata.NewSandboxStore(store.NewMetadataStore()),
+		sandboxStore:        sandboxstore.NewStore(),
-		containerStore:      metadata.NewContainerStore(store.NewMetadataStore()),
+		containerStore:      containerstore.NewStore(),
-		imageMetadataStore:  metadata.NewImageMetadataStore(store.NewMetadataStore()),
+		imageStore:          imagestore.NewStore(),
 		sandboxNameIndex:    registrar.NewRegistrar(),
 		containerNameIndex:  registrar.NewRegistrar(),
 		containerService:    client.ContainerService(),

pkg/server/service_test.go

@@ -29,12 +29,13 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
-	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata/store"
 	ostesting "github.com/kubernetes-incubator/cri-containerd/pkg/os/testing"
 	"github.com/kubernetes-incubator/cri-containerd/pkg/registrar"
 	agentstesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/agents/testing"
 	servertesting "github.com/kubernetes-incubator/cri-containerd/pkg/server/testing"
+	containerstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/container"
+	imagestore "github.com/kubernetes-incubator/cri-containerd/pkg/store/image"
+	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
 type nopReadWriteCloser struct{}
@@ -58,10 +59,10 @@ func newTestCRIContainerdService() *criContainerdService {
 		os:                 ostesting.NewFakeOS(),
 		rootDir:            testRootDir,
 		sandboxImage:       testSandboxImage,
-		sandboxStore:       metadata.NewSandboxStore(store.NewMetadataStore()),
+		sandboxStore:       sandboxstore.NewStore(),
-		imageMetadataStore: metadata.NewImageMetadataStore(store.NewMetadataStore()),
+		imageStore:         imagestore.NewStore(),
 		sandboxNameIndex:   registrar.NewRegistrar(),
-		containerStore:     metadata.NewContainerStore(store.NewMetadataStore()),
+		containerStore:     containerstore.NewStore(),
 		containerNameIndex: registrar.NewRegistrar(),
 		taskService:        servertesting.NewFakeExecutionClient(),
 		containerService:   servertesting.NewFakeContainersClient(),
@@ -88,11 +89,11 @@ func TestSandboxOperations(t *testing.T) {
 		return nopReadWriteCloser{}, nil
 	}
 	// Insert sandbox image metadata.
-	assert.NoError(t, c.imageMetadataStore.Create(metadata.ImageMetadata{
+	c.imageStore.Add(imagestore.Image{
 		ID:      testSandboxImage,
 		ChainID: "test-chain-id",
 		Config:  &imagespec.ImageConfig{Entrypoint: []string{"/pause"}},
-	}))
+	})
 
 	config := &runtime.PodSandboxConfig{
 		Metadata: &runtime.PodSandboxMetadata{