Merge containerd/cri into containerd/containerd

Signed-off-by: Derek McGowan <derek@mcg.dev>

pkg/annotations/annotations.go

@@ -0,0 +1,50 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package annotations
+
+// ContainerType values
+// Following OCI annotations are used by katacontainers now.
+// We'll switch to standard secure pod API after it is defined in CRI.
+const (
+	// ContainerTypeSandbox represents a pod sandbox container
+	ContainerTypeSandbox = "sandbox"
+
+	// ContainerTypeContainer represents a container running within a pod
+	ContainerTypeContainer = "container"
+
+	// ContainerType is the container type (sandbox or container) annotation
+	ContainerType = "io.kubernetes.cri.container-type"
+
+	// SandboxID is the sandbox ID annotation
+	SandboxID = "io.kubernetes.cri.sandbox-id"
+
+	// SandboxLogDir is the pod log directory annotation.
+	// If the sandbox needs to generate any log, it will put it into this directory.
+	// Kubelet will be responsible for:
+	// 1) Monitoring the disk usage of the log, and including it as part of the pod
+	// ephemeral storage usage.
+	// 2) Cleaning up the logs when the pod is deleted.
+	// NOTE: Kubelet is not responsible for rotating the logs.
+	SandboxLogDir = "io.kubernetes.cri.sandbox-log-directory"
+
+	// UntrustedWorkload is the sandbox annotation for untrusted workload. Untrusted
+	// workload can only run on dedicated runtime for untrusted workload.
+	UntrustedWorkload = "io.kubernetes.cri.untrusted-workload"
+
+	// containerName is the name of the container in the pod
+	ContainerName = "io.kubernetes.cri.container-name"
+)

pkg/api/runtimeoptions/v1/api.pb.go

@@ -0,0 +1,394 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: api.proto
+
+/*
+	Package cri_runtimeoptions_v1 is a generated protocol buffer package.
+
+	It is generated from these files:
+		api.proto
+
+	It has these top-level messages:
+		Options
+*/
+package cri_runtimeoptions_v1
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import strings "strings"
+import reflect "reflect"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type Options struct {
+	// TypeUrl specifies the type of the content inside the config file.
+	TypeUrl string `protobuf:"bytes,1,opt,name=type_url,json=typeUrl,proto3" json:"type_url,omitempty"`
+	// ConfigPath specifies the filesystem location of the config file
+	// used by the runtime.
+	ConfigPath string `protobuf:"bytes,2,opt,name=config_path,json=configPath,proto3" json:"config_path,omitempty"`
+}
+
+func (m *Options) Reset()                    { *m = Options{} }
+func (*Options) ProtoMessage()               {}
+func (*Options) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} }
+
+func (m *Options) GetTypeUrl() string {
+	if m != nil {
+		return m.TypeUrl
+	}
+	return ""
+}
+
+func (m *Options) GetConfigPath() string {
+	if m != nil {
+		return m.ConfigPath
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*Options)(nil), "cri.runtimeoptions.v1.Options")
+}
+func (m *Options) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Options) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.TypeUrl) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintApi(dAtA, i, uint64(len(m.TypeUrl)))
+		i += copy(dAtA[i:], m.TypeUrl)
+	}
+	if len(m.ConfigPath) > 0 {
+		dAtA[i] = 0x12
+		i++
+		i = encodeVarintApi(dAtA, i, uint64(len(m.ConfigPath)))
+		i += copy(dAtA[i:], m.ConfigPath)
+	}
+	return i, nil
+}
+
+func encodeVarintApi(dAtA []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return offset + 1
+}
+func (m *Options) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.TypeUrl)
+	if l > 0 {
+		n += 1 + l + sovApi(uint64(l))
+	}
+	l = len(m.ConfigPath)
+	if l > 0 {
+		n += 1 + l + sovApi(uint64(l))
+	}
+	return n
+}
+
+func sovApi(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozApi(x uint64) (n int) {
+	return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (this *Options) String() string {
+	if this == nil {
+		return "nil"
+	}
+	s := strings.Join([]string{`&Options{`,
+		`TypeUrl:` + fmt.Sprintf("%v", this.TypeUrl) + `,`,
+		`ConfigPath:` + fmt.Sprintf("%v", this.ConfigPath) + `,`,
+		`}`,
+	}, "")
+	return s
+}
+func valueToStringApi(v interface{}) string {
+	rv := reflect.ValueOf(v)
+	if rv.IsNil() {
+		return "nil"
+	}
+	pv := reflect.Indirect(rv).Interface()
+	return fmt.Sprintf("*%v", pv)
+}
+func (m *Options) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Options: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Options: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field TypeUrl", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.TypeUrl = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ConfigPath", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthApi
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ConfigPath = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipApi(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthApi
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipApi(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowApi
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowApi
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthApi
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowApi
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipApi(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowApi   = fmt.Errorf("proto: integer overflow")
+)
+
+func init() { proto.RegisterFile("api.proto", fileDescriptorApi) }
+
+var fileDescriptorApi = []byte{
+	// 183 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2c, 0xc8, 0xd4,
+	0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x4d, 0x2e, 0xca, 0xd4, 0x2b, 0x2a, 0xcd, 0x2b, 0xc9,
+	0xcc, 0x4d, 0xcd, 0x2f, 0x28, 0xc9, 0xcc, 0xcf, 0x2b, 0xd6, 0x2b, 0x33, 0x94, 0xd2, 0x4d, 0xcf,
+	0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f, 0xcf, 0xd7, 0x07, 0xab,
+	0x4e, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0x62, 0x8a, 0x92, 0x2b, 0x17, 0xbb, 0x3f,
+	0x44, 0xb3, 0x90, 0x24, 0x17, 0x47, 0x49, 0x65, 0x41, 0x6a, 0x7c, 0x69, 0x51, 0x8e, 0x04, 0xa3,
+	0x02, 0xa3, 0x06, 0x67, 0x10, 0x3b, 0x88, 0x1f, 0x5a, 0x94, 0x23, 0x24, 0xcf, 0xc5, 0x9d, 0x9c,
+	0x9f, 0x97, 0x96, 0x99, 0x1e, 0x5f, 0x90, 0x58, 0x92, 0x21, 0xc1, 0x04, 0x96, 0xe5, 0x82, 0x08,
+	0x05, 0x24, 0x96, 0x64, 0x38, 0xc9, 0x9c, 0x78, 0x28, 0xc7, 0x78, 0xe3, 0xa1, 0x1c, 0x43, 0xc3,
+	0x23, 0x39, 0xc6, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71,
+	0xc2, 0x63, 0x39, 0x86, 0x24, 0x36, 0xb0, 0x5d, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x07,
+	0x00, 0xf2, 0x18, 0xbe, 0x00, 0x00, 0x00,
+}

pkg/api/runtimeoptions/v1/api.proto

@@ -0,0 +1,22 @@
+// To regenerate api.pb.go run `make proto`
+syntax = "proto3";
+
+package cri.runtimeoptions.v1;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+option (gogoproto.goproto_getters_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unrecognized_all) = false;
+
+message Options {
+	// TypeUrl specifies the type of the content inside the config file.
+	string type_url = 1;
+	// ConfigPath specifies the filesystem location of the config file
+	// used by the runtime.
+	string config_path = 2;
+}

pkg/atomic/atomic_boolean.go

@@ -0,0 +1,54 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package atomic
+
+import "sync/atomic"
+
+// Bool is an atomic Boolean,
+// Its methods are all atomic, thus safe to be called by
+// multiple goroutines simultaneously.
+type Bool interface {
+	Set()
+	Unset()
+	IsSet() bool
+}
+
+// NewBool creates an Bool with given default value
+func NewBool(ok bool) Bool {
+	ab := new(atomicBool)
+	if ok {
+		ab.Set()
+	}
+	return ab
+}
+
+type atomicBool int32
+
+// Set sets the Boolean to true
+func (ab *atomicBool) Set() {
+	atomic.StoreInt32((*int32)(ab), 1)
+}
+
+// Unset sets the Boolean to false
+func (ab *atomicBool) Unset() {
+	atomic.StoreInt32((*int32)(ab), 0)
+}
+
+// IsSet returns whether the Boolean is true
+func (ab *atomicBool) IsSet() bool {
+	return atomic.LoadInt32((*int32)(ab)) == 1
+}

pkg/atomic/atomic_boolean_test.go

@@ -0,0 +1,32 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package atomic
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBoolean(t *testing.T) {
+	ab := NewBool(true)
+	assert.True(t, ab.IsSet())
+	ab.Unset()
+	assert.False(t, ab.IsSet())
+	ab.Set()
+	assert.True(t, ab.IsSet())
+}

pkg/config/config.go

@@ -0,0 +1,369 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"context"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/plugin"
+	"github.com/pkg/errors"
+)
+
+// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
+// and a runtime for untrusted worload.
+type Runtime struct {
+	// Type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+	Type string `toml:"runtime_type" json:"runtimeType"`
+	// Engine is the name of the runtime engine used by containerd.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
+	Engine string `toml:"runtime_engine" json:"runtimeEngine"`
+	// PodAnnotations is a list of pod annotations passed to both pod sandbox as well as
+	// container OCI annotations.
+	PodAnnotations []string `toml:"pod_annotations" json:"PodAnnotations"`
+	// ContainerAnnotations is a list of container annotations passed through to the OCI config of the containers.
+	// Container annotations in CRI are usually generated by other Kubernetes node components (i.e., not users).
+	// Currently, only device plugins populate the annotations.
+	ContainerAnnotations []string `toml:"container_annotations" json:"ContainerAnnotations"`
+	// Root is the directory used by containerd for runtime state.
+	// DEPRECATED: use Options instead. Remove when shim v1 is deprecated.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	Root string `toml:"runtime_root" json:"runtimeRoot"`
+	// Options are config options for the runtime. If options is loaded
+	// from toml config, it will be toml.Primitive.
+	Options *toml.Primitive `toml:"options" json:"options"`
+	// PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the
+	// runtime spec when the container is privileged. Defaults to false.
+	PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
+	// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
+	BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
+}
+
+// ContainerdConfig contains toml config related to containerd
+type ContainerdConfig struct {
+	// Snapshotter is the snapshotter used by containerd.
+	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
+	// DefaultRuntimeName is the default runtime name to use from the runtimes table.
+	DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"`
+	// DefaultRuntime is the default runtime to use in containerd.
+	// This runtime is used when no runtime handler (or the empty string) is provided.
+	// DEPRECATED: use DefaultRuntimeName instead. Remove in containerd 1.4.
+	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"`
+	// UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it.
+	// DEPRECATED: use `untrusted` runtime in Runtimes instead. Remove in containerd 1.4.
+	UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"`
+	// Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime
+	// configurations, to the matching configurations.
+	Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
+	// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	NoPivot bool `toml:"no_pivot" json:"noPivot"`
+
+	// DisableSnapshotAnnotations disables to pass additional annotations (image
+	// related information) to snapshotters. These annotations are required by
+	// stargz snapshotter (https://github.com/containerd/stargz-snapshotter).
+	DisableSnapshotAnnotations bool `toml:"disable_snapshot_annotations" json:"disableSnapshotAnnotations"`
+
+	// DiscardUnpackedLayers is a boolean flag to specify whether to allow GC to
+	// remove layers from the content store after successfully unpacking these
+	// layers to the snapshotter.
+	DiscardUnpackedLayers bool `toml:"discard_unpacked_layers" json:"discardUnpackedLayers"`
+}
+
+// CniConfig contains toml config related to cni
+type CniConfig struct {
+	// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
+	NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
+	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
+	NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
+	// NetworkPluginMaxConfNum is the max number of plugin config files that will
+	// be loaded from the cni config directory by go-cni. Set the value to 0 to
+	// load all config files (no arbitrary limit). The legacy default value is 1.
+	NetworkPluginMaxConfNum int `toml:"max_conf_num" json:"maxConfNum"`
+	// NetworkPluginConfTemplate is the file path of golang template used to generate
+	// cni config.
+	// When it is set, containerd will get cidr(s) from kubelet to replace {{.PodCIDR}},
+	// {{.PodCIDRRanges}} or {{.Routes}} in the template, and write the config into
+	// NetworkPluginConfDir.
+	// Ideally the cni config should be placed by system admin or cni daemon like calico,
+	// weaveworks etc. However, there are still users using kubenet
+	// (https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet)
+	// today, who don't have a cni daemonset in production. NetworkPluginConfTemplate is
+	// a temporary backward-compatible solution for them.
+	// TODO(random-liu): Deprecate this option when kubenet is deprecated.
+	NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
+}
+
+// Mirror contains the config related to the registry mirror
+type Mirror struct {
+	// Endpoints are endpoints for a namespace. CRI plugin will try the endpoints
+	// one by one until a working one is found. The endpoint must be a valid url
+	// with host specified.
+	// The scheme, host and path from the endpoint URL will be used.
+	Endpoints []string `toml:"endpoint" json:"endpoint"`
+}
+
+// AuthConfig contains the config related to authentication to a specific registry
+type AuthConfig struct {
+	// Username is the username to login the registry.
+	Username string `toml:"username" json:"username"`
+	// Password is the password to login the registry.
+	Password string `toml:"password" json:"password"`
+	// Auth is a base64 encoded string from the concatenation of the username,
+	// a colon, and the password.
+	Auth string `toml:"auth" json:"auth"`
+	// IdentityToken is used to authenticate the user and get
+	// an access token for the registry.
+	IdentityToken string `toml:"identitytoken" json:"identitytoken"`
+}
+
+// TLSConfig contains the CA/Cert/Key used for a registry
+type TLSConfig struct {
+	InsecureSkipVerify bool   `toml:"insecure_skip_verify" json:"insecure_skip_verify"`
+	CAFile             string `toml:"ca_file" json:"caFile"`
+	CertFile           string `toml:"cert_file" json:"certFile"`
+	KeyFile            string `toml:"key_file" json:"keyFile"`
+}
+
+// Registry is registry settings configured
+type Registry struct {
+	// Mirrors are namespace to mirror mapping for all namespaces.
+	Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
+	// Configs are configs for each registry.
+	// The key is the domain name or IP of the registry.
+	Configs map[string]RegistryConfig `toml:"configs" json:"configs"`
+
+	// Auths are registry endpoint to auth config mapping. The registry endpoint must
+	// be a valid url with host specified.
+	// DEPRECATED: Use Configs instead. Remove in containerd 1.4.
+	Auths map[string]AuthConfig `toml:"auths" json:"auths"`
+	// Headers adds additional HTTP headers that get sent to all registries
+	Headers map[string][]string `toml:"headers" json:"headers"`
+}
+
+// RegistryConfig contains configuration used to communicate with the registry.
+type RegistryConfig struct {
+	// Auth contains information to authenticate to the registry.
+	Auth *AuthConfig `toml:"auth" json:"auth"`
+	// TLS is a pair of CA/Cert/Key which then are used when creating the transport
+	// that communicates with the registry.
+	TLS *TLSConfig `toml:"tls" json:"tls"`
+}
+
+// ImageDecryption contains configuration to handling decryption of encrypted container images.
+type ImageDecryption struct {
+	// KeyModel specifies the trust model of where keys should reside.
+	//
+	// Details of field usage can be found in:
+	// https://github.com/containerd/cri/tree/master/docs/config.md
+	//
+	// Details of key models can be found in:
+	// https://github.com/containerd/cri/tree/master/docs/decryption.md
+	KeyModel string `toml:"key_model" json:"keyModel"`
+}
+
+// PluginConfig contains toml config related to CRI plugin,
+// it is a subset of Config.
+type PluginConfig struct {
+	// ContainerdConfig contains config related to containerd
+	ContainerdConfig `toml:"containerd" json:"containerd"`
+	// CniConfig contains config related to cni
+	CniConfig `toml:"cni" json:"cni"`
+	// Registry contains config related to the registry
+	Registry Registry `toml:"registry" json:"registry"`
+	// ImageDecryption contains config related to handling decryption of encrypted container images
+	ImageDecryption `toml:"image_decryption" json:"imageDecryption"`
+	// DisableTCPService disables serving CRI on the TCP server.
+	DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
+	// StreamServerAddress is the ip address streaming server is listening on.
+	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
+	// StreamServerPort is the port streaming server is listening on.
+	StreamServerPort string `toml:"stream_server_port" json:"streamServerPort"`
+	// StreamIdleTimeout is the maximum time a streaming connection
+	// can be idle before the connection is automatically closed.
+	// The string is in the golang duration format, see:
+	//   https://golang.org/pkg/time/#ParseDuration
+	StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"`
+	// EnableSelinux indicates to enable the selinux support.
+	EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux"`
+	// SelinuxCategoryRange allows the upper bound on the category range to be set.
+	// If not specified or set to 0, defaults to 1024 from the selinux package.
+	SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"`
+	// SandboxImage is the image used by sandbox container.
+	SandboxImage string `toml:"sandbox_image" json:"sandboxImage"`
+	// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
+	StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
+	// SystemdCgroup enables systemd cgroup support.
+	// This only works for runtime type "io.containerd.runtime.v1.linux".
+	// DEPRECATED: config runc runtime handler instead. Remove when shim v1 is deprecated.
+	SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
+	// EnableTLSStreaming indicates to enable the TLS streaming support.
+	EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`
+	// X509KeyPairStreaming is a x509 key pair used for TLS streaming
+	X509KeyPairStreaming `toml:"x509_key_pair_streaming" json:"x509KeyPairStreaming"`
+	// MaxContainerLogLineSize is the maximum log line size in bytes for a container.
+	// Log line longer than the limit will be split into multiple lines. Non-positive
+	// value means no limit.
+	MaxContainerLogLineSize int `toml:"max_container_log_line_size" json:"maxContainerLogSize"`
+	// DisableCgroup indicates to disable the cgroup support.
+	// This is useful when the containerd does not have permission to access cgroup.
+	DisableCgroup bool `toml:"disable_cgroup" json:"disableCgroup"`
+	// DisableApparmor indicates to disable the apparmor support.
+	// This is useful when the containerd does not have permission to access Apparmor.
+	DisableApparmor bool `toml:"disable_apparmor" json:"disableApparmor"`
+	// RestrictOOMScoreAdj indicates to limit the lower bound of OOMScoreAdj to the containerd's
+	// current OOMScoreADj.
+	// This is useful when the containerd does not have permission to decrease OOMScoreAdj.
+	RestrictOOMScoreAdj bool `toml:"restrict_oom_score_adj" json:"restrictOOMScoreAdj"`
+	// MaxConcurrentDownloads restricts the number of concurrent downloads for each image.
+	MaxConcurrentDownloads int `toml:"max_concurrent_downloads" json:"maxConcurrentDownloads"`
+	// DisableProcMount disables Kubernetes ProcMount support. This MUST be set to `true`
+	// when using containerd with Kubernetes <=1.11.
+	DisableProcMount bool `toml:"disable_proc_mount" json:"disableProcMount"`
+	// UnsetSeccompProfile is the profile containerd/cri will use If the provided seccomp profile is
+	// unset (`""`) for a container (default is `unconfined`)
+	UnsetSeccompProfile string `toml:"unset_seccomp_profile" json:"unsetSeccompProfile"`
+	// TolerateMissingHugetlbController if set to false will error out on create/update
+	// container requests with huge page limits if the cgroup controller for hugepages is not present.
+	// This helps with supporting Kubernetes <=1.18 out of the box. (default is `true`)
+	TolerateMissingHugetlbController bool `toml:"tolerate_missing_hugetlb_controller" json:"tolerateMissingHugetlbController"`
+	// DisableHugetlbController indicates to silently disable the hugetlb controller, even when it is
+	// present in /sys/fs/cgroup/cgroup.controllers.
+	// This helps with running rootless mode + cgroup v2 + systemd but without hugetlb delegation.
+	DisableHugetlbController bool `toml:"disable_hugetlb_controller" json:"disableHugetlbController"`
+	// IgnoreImageDefinedVolumes ignores volumes defined by the image. Useful for better resource
+	// isolation, security and early detection of issues in the mount configuration when using
+	// ReadOnlyRootFilesystem since containers won't silently mount a temporary volume.
+	IgnoreImageDefinedVolumes bool `toml:"ignore_image_defined_volumes" json:"ignoreImageDefinedVolumes"`
+}
+
+// X509KeyPairStreaming contains the x509 configuration for streaming
+type X509KeyPairStreaming struct {
+	// TLSCertFile is the path to a certificate file
+	TLSCertFile string `toml:"tls_cert_file" json:"tlsCertFile"`
+	// TLSKeyFile is the path to a private key file
+	TLSKeyFile string `toml:"tls_key_file" json:"tlsKeyFile"`
+}
+
+// Config contains all configurations for cri server.
+type Config struct {
+	// PluginConfig is the config for CRI plugin.
+	PluginConfig
+	// ContainerdRootDir is the root directory path for containerd.
+	ContainerdRootDir string `json:"containerdRootDir"`
+	// ContainerdEndpoint is the containerd endpoint path.
+	ContainerdEndpoint string `json:"containerdEndpoint"`
+	// RootDir is the root directory path for managing cri plugin files
+	// (metadata checkpoint etc.)
+	RootDir string `json:"rootDir"`
+	// StateDir is the root directory path for managing volatile pod/container data
+	StateDir string `json:"stateDir"`
+}
+
+const (
+	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
+	RuntimeUntrusted = "untrusted"
+	// RuntimeDefault is the implicit runtime defined for ContainerdConfig.DefaultRuntime
+	RuntimeDefault = "default"
+	// KeyModelNode is the key model where key for encrypted images reside
+	// on the worker nodes
+	KeyModelNode = "node"
+)
+
+// ValidatePluginConfig validates the given plugin configuration.
+func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error {
+	if c.ContainerdConfig.Runtimes == nil {
+		c.ContainerdConfig.Runtimes = make(map[string]Runtime)
+	}
+
+	// Validation for deprecated untrusted_workload_runtime.
+	if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
+		log.G(ctx).Warning("`untrusted_workload_runtime` is deprecated, please use `untrusted` runtime in `runtimes` instead")
+		if _, ok := c.ContainerdConfig.Runtimes[RuntimeUntrusted]; ok {
+			return errors.Errorf("conflicting definitions: configuration includes both `untrusted_workload_runtime` and `runtimes[%q]`", RuntimeUntrusted)
+		}
+		c.ContainerdConfig.Runtimes[RuntimeUntrusted] = c.ContainerdConfig.UntrustedWorkloadRuntime
+	}
+
+	// Validation for deprecated default_runtime field.
+	if c.ContainerdConfig.DefaultRuntime.Type != "" {
+		log.G(ctx).Warning("`default_runtime` is deprecated, please use `default_runtime_name` to reference the default configuration you have defined in `runtimes`")
+		c.ContainerdConfig.DefaultRuntimeName = RuntimeDefault
+		c.ContainerdConfig.Runtimes[RuntimeDefault] = c.ContainerdConfig.DefaultRuntime
+	}
+
+	// Validation for default_runtime_name
+	if c.ContainerdConfig.DefaultRuntimeName == "" {
+		return errors.New("`default_runtime_name` is empty")
+	}
+	if _, ok := c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName]; !ok {
+		return errors.New("no corresponding runtime configured in `runtimes` for `default_runtime_name`")
+	}
+
+	// Validation for deprecated runtime options.
+	if c.SystemdCgroup {
+		if c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName].Type != plugin.RuntimeLinuxV1 {
+			return errors.Errorf("`systemd_cgroup` only works for runtime %s", plugin.RuntimeLinuxV1)
+		}
+		log.G(ctx).Warning("`systemd_cgroup` is deprecated, please use runtime `options` instead")
+	}
+	if c.NoPivot {
+		if c.ContainerdConfig.Runtimes[c.ContainerdConfig.DefaultRuntimeName].Type != plugin.RuntimeLinuxV1 {
+			return errors.Errorf("`no_pivot` only works for runtime %s", plugin.RuntimeLinuxV1)
+		}
+		// NoPivot can't be deprecated yet, because there is no alternative config option
+		// for `io.containerd.runtime.v1.linux`.
+	}
+	for _, r := range c.ContainerdConfig.Runtimes {
+		if r.Engine != "" {
+			if r.Type != plugin.RuntimeLinuxV1 {
+				return errors.Errorf("`runtime_engine` only works for runtime %s", plugin.RuntimeLinuxV1)
+			}
+			log.G(ctx).Warning("`runtime_engine` is deprecated, please use runtime `options` instead")
+		}
+		if r.Root != "" {
+			if r.Type != plugin.RuntimeLinuxV1 {
+				return errors.Errorf("`runtime_root` only works for runtime %s", plugin.RuntimeLinuxV1)
+			}
+			log.G(ctx).Warning("`runtime_root` is deprecated, please use runtime `options` instead")
+		}
+	}
+
+	// Validation for deprecated auths options and mapping it to configs.
+	if len(c.Registry.Auths) != 0 {
+		if c.Registry.Configs == nil {
+			c.Registry.Configs = make(map[string]RegistryConfig)
+		}
+		for endpoint, auth := range c.Registry.Auths {
+			config := c.Registry.Configs[endpoint]
+			config.Auth = &auth
+			c.Registry.Configs[endpoint] = config
+		}
+		log.G(ctx).Warning("`auths` is deprecated, please use registry`configs` instead")
+	}
+
+	// Validation for stream_idle_timeout
+	if c.StreamIdleTimeout != "" {
+		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {
+			return errors.Wrap(err, "invalid stream idle timeout")
+		}
+	}
+	return nil
+}

pkg/config/config_test.go

@@ -0,0 +1,334 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/containerd/containerd/plugin"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateConfig(t *testing.T) {
+	for desc, test := range map[string]struct {
+		config      *PluginConfig
+		expectedErr string
+		expected    *PluginConfig
+	}{
+		"deprecated untrusted_workload_runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					UntrustedWorkloadRuntime: Runtime{
+						Type: "untrusted",
+					},
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					UntrustedWorkloadRuntime: Runtime{
+						Type: "untrusted",
+					},
+					Runtimes: map[string]Runtime{
+						RuntimeUntrusted: {
+							Type: "untrusted",
+						},
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+			},
+		},
+		"both untrusted_workload_runtime and runtime[untrusted]": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					UntrustedWorkloadRuntime: Runtime{
+						Type: "untrusted-1",
+					},
+					Runtimes: map[string]Runtime{
+						RuntimeUntrusted: {
+							Type: "untrusted-2",
+						},
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+			},
+			expectedErr: fmt.Sprintf("conflicting definitions: configuration includes both `untrusted_workload_runtime` and `runtimes[%q]`", RuntimeUntrusted),
+		},
+		"deprecated default_runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntime: Runtime{
+						Type: "default",
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntime: Runtime{
+						Type: "default",
+					},
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+			},
+		},
+		"no default_runtime_name": {
+			config:      &PluginConfig{},
+			expectedErr: "`default_runtime_name` is empty",
+		},
+		"no runtime[default_runtime_name]": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+				},
+			},
+			expectedErr: "no corresponding runtime configured in `runtimes` for `default_runtime_name`",
+		},
+		"deprecated systemd_cgroup for v1 runtime": {
+			config: &PluginConfig{
+				SystemdCgroup: true,
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				SystemdCgroup: true,
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+		},
+		"deprecated systemd_cgroup for v2 runtime": {
+			config: &PluginConfig{
+				SystemdCgroup: true,
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeRuncV1,
+						},
+					},
+				},
+			},
+			expectedErr: fmt.Sprintf("`systemd_cgroup` only works for runtime %s", plugin.RuntimeLinuxV1),
+		},
+		"no_pivot for v1 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					NoPivot:            true,
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					NoPivot:            true,
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+		},
+		"no_pivot for v2 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					NoPivot:            true,
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeRuncV1,
+						},
+					},
+				},
+			},
+			expectedErr: fmt.Sprintf("`no_pivot` only works for runtime %s", plugin.RuntimeLinuxV1),
+		},
+		"deprecated runtime_engine for v1 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Engine: "runc",
+							Type:   plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Engine: "runc",
+							Type:   plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+		},
+		"deprecated runtime_engine for v2 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Engine: "runc",
+							Type:   plugin.RuntimeRuncV1,
+						},
+					},
+				},
+			},
+			expectedErr: fmt.Sprintf("`runtime_engine` only works for runtime %s", plugin.RuntimeLinuxV1),
+		},
+		"deprecated runtime_root for v1 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Root: "/run/containerd/runc",
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Root: "/run/containerd/runc",
+							Type: plugin.RuntimeLinuxV1,
+						},
+					},
+				},
+			},
+		},
+		"deprecated runtime_root for v2 runtime": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Root: "/run/containerd/runc",
+							Type: plugin.RuntimeRuncV1,
+						},
+					},
+				},
+			},
+			expectedErr: fmt.Sprintf("`runtime_root` only works for runtime %s", plugin.RuntimeLinuxV1),
+		},
+		"deprecated auths": {
+			config: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeRuncV1,
+						},
+					},
+				},
+				Registry: Registry{
+					Auths: map[string]AuthConfig{
+						"https://gcr.io": {Username: "test"},
+					},
+				},
+			},
+			expected: &PluginConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: plugin.RuntimeRuncV1,
+						},
+					},
+				},
+				Registry: Registry{
+					Configs: map[string]RegistryConfig{
+						"https://gcr.io": {
+							Auth: &AuthConfig{
+								Username: "test",
+							},
+						},
+					},
+					Auths: map[string]AuthConfig{
+						"https://gcr.io": {Username: "test"},
+					},
+				},
+			},
+		},
+		"invalid stream_idle_timeout": {
+			config: &PluginConfig{
+				StreamIdleTimeout: "invalid",
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							Type: "default",
+						},
+					},
+				},
+			},
+			expectedErr: "invalid stream idle timeout",
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			err := ValidatePluginConfig(context.Background(), test.config)
+			if test.expectedErr != "" {
+				assert.Contains(t, err.Error(), test.expectedErr)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.expected, test.config)
+			}
+		})
+	}
+}

pkg/config/config_unix.go

@@ -0,0 +1,75 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd"
+	"github.com/containerd/cri/pkg/streaming"
+)
+
+// DefaultConfig returns default configurations of cri plugin.
+func DefaultConfig() PluginConfig {
+	return PluginConfig{
+		CniConfig: CniConfig{
+			NetworkPluginBinDir:       "/opt/cni/bin",
+			NetworkPluginConfDir:      "/etc/cni/net.d",
+			NetworkPluginMaxConfNum:   1, // only one CNI plugin config file will be loaded
+			NetworkPluginConfTemplate: "",
+		},
+		ContainerdConfig: ContainerdConfig{
+			Snapshotter:        containerd.DefaultSnapshotter,
+			DefaultRuntimeName: "runc",
+			NoPivot:            false,
+			Runtimes: map[string]Runtime{
+				"runc": {
+					Type:    "io.containerd.runc.v2",
+					Options: new(toml.Primitive),
+				},
+			},
+		},
+		DisableTCPService:    true,
+		StreamServerAddress:  "127.0.0.1",
+		StreamServerPort:     "0",
+		StreamIdleTimeout:    streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
+		EnableSelinux:        false,
+		SelinuxCategoryRange: 1024,
+		EnableTLSStreaming:   false,
+		X509KeyPairStreaming: X509KeyPairStreaming{
+			TLSKeyFile:  "",
+			TLSCertFile: "",
+		},
+		SandboxImage:            "k8s.gcr.io/pause:3.2",
+		StatsCollectPeriod:      10,
+		SystemdCgroup:           false,
+		MaxContainerLogLineSize: 16 * 1024,
+		Registry: Registry{
+			Mirrors: map[string]Mirror{
+				"docker.io": {
+					Endpoints: []string{"https://registry-1.docker.io"},
+				},
+			},
+		},
+		MaxConcurrentDownloads:           3,
+		DisableProcMount:                 false,
+		TolerateMissingHugetlbController: true,
+		DisableHugetlbController:         true,
+		IgnoreImageDefinedVolumes:        false,
+	}
+}

pkg/config/config_windows.go

@@ -0,0 +1,71 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/cri/pkg/streaming"
+)
+
+// DefaultConfig returns default configurations of cri plugin.
+func DefaultConfig() PluginConfig {
+	return PluginConfig{
+		CniConfig: CniConfig{
+			NetworkPluginBinDir:       filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin"),
+			NetworkPluginConfDir:      filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "conf"),
+			NetworkPluginMaxConfNum:   1,
+			NetworkPluginConfTemplate: "",
+		},
+		ContainerdConfig: ContainerdConfig{
+			Snapshotter:        containerd.DefaultSnapshotter,
+			DefaultRuntimeName: "runhcs-wcow-process",
+			NoPivot:            false,
+			Runtimes: map[string]Runtime{
+				"runhcs-wcow-process": {
+					Type: "io.containerd.runhcs.v1",
+				},
+			},
+		},
+		DisableTCPService:   true,
+		StreamServerAddress: "127.0.0.1",
+		StreamServerPort:    "0",
+		StreamIdleTimeout:   streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
+		EnableTLSStreaming:  false,
+		X509KeyPairStreaming: X509KeyPairStreaming{
+			TLSKeyFile:  "",
+			TLSCertFile: "",
+		},
+		SandboxImage:            "mcr.microsoft.com/oss/kubernetes/pause:1.4.0",
+		StatsCollectPeriod:      10,
+		MaxContainerLogLineSize: 16 * 1024,
+		Registry: Registry{
+			Mirrors: map[string]Mirror{
+				"docker.io": {
+					Endpoints: []string{"https://registry-1.docker.io"},
+				},
+			},
+		},
+		MaxConcurrentDownloads:    3,
+		IgnoreImageDefinedVolumes: false,
+		// TODO(windows): Add platform specific config, so that most common defaults can be shared.
+	}
+}

pkg/constants/constants.go

@@ -0,0 +1,26 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package constants
+
+// TODO(random-liu): Merge annotations package into this package.
+
+const (
+	// K8sContainerdNamespace is the namespace we use to connect containerd.
+	K8sContainerdNamespace = "k8s.io"
+	// CRIVersion is the CRI version supported by the CRI plugin.
+	CRIVersion = "v1alpha2"
+)

pkg/containerd/opts/container.go

@@ -0,0 +1,118 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/continuity/fs"
+	"github.com/pkg/errors"
+)
+
+// WithNewSnapshot wraps `containerd.WithNewSnapshot` so that if creating the
+// snapshot fails we make sure the image is actually unpacked and and retry.
+func WithNewSnapshot(id string, i containerd.Image) containerd.NewContainerOpts {
+	f := containerd.WithNewSnapshot(id, i)
+	return func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
+		if err := f(ctx, client, c); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return err
+			}
+
+			if err := i.Unpack(ctx, c.Snapshotter); err != nil {
+				return errors.Wrap(err, "error unpacking image")
+			}
+			return f(ctx, client, c)
+		}
+		return nil
+	}
+}
+
+// WithVolumes copies ownership of volume in rootfs to its corresponding host path.
+// It doesn't update runtime spec.
+// The passed in map is a host path to container path map for all volumes.
+func WithVolumes(volumeMounts map[string]string) containerd.NewContainerOpts {
+	return func(ctx context.Context, client *containerd.Client, c *containers.Container) (err error) {
+		if c.Snapshotter == "" {
+			return errors.New("no snapshotter set for container")
+		}
+		if c.SnapshotKey == "" {
+			return errors.New("rootfs not created for container")
+		}
+		snapshotter := client.SnapshotService(c.Snapshotter)
+		mounts, err := snapshotter.Mounts(ctx, c.SnapshotKey)
+		if err != nil {
+			return err
+		}
+		root, err := ioutil.TempDir("", "ctd-volume")
+		if err != nil {
+			return err
+		}
+		// We change RemoveAll to Remove so that we either leak a temp dir
+		// if it fails but not RM snapshot data.
+		// refer to https://github.com/containerd/containerd/pull/1868
+		// https://github.com/containerd/containerd/pull/1785
+		defer os.Remove(root) // nolint: errcheck
+		if err := mount.All(mounts, root); err != nil {
+			return errors.Wrap(err, "failed to mount")
+		}
+		defer func() {
+			if uerr := mount.Unmount(root, 0); uerr != nil {
+				log.G(ctx).WithError(uerr).Errorf("Failed to unmount snapshot %q", c.SnapshotKey)
+				if err == nil {
+					err = uerr
+				}
+			}
+		}()
+
+		for host, volume := range volumeMounts {
+			src := filepath.Join(root, volume)
+			if _, err := os.Stat(src); err != nil {
+				if os.IsNotExist(err) {
+					// Skip copying directory if it does not exist.
+					continue
+				}
+				return errors.Wrap(err, "stat volume in rootfs")
+			}
+			if err := copyExistingContents(src, host); err != nil {
+				return errors.Wrap(err, "taking runtime copy of volume")
+			}
+		}
+		return nil
+	}
+}
+
+// copyExistingContents copies from the source to the destination and
+// ensures the ownership is appropriately set.
+func copyExistingContents(source, destination string) error {
+	dstList, err := ioutil.ReadDir(destination)
+	if err != nil {
+		return err
+	}
+	if len(dstList) != 0 {
+		return errors.Errorf("volume at %q is not initially empty", destination)
+	}
+	return fs.CopyDir(destination, source)
+}

pkg/containerd/opts/spec.go

@@ -0,0 +1,113 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// DefaultSandboxCPUshares is default cpu shares for sandbox container.
+// TODO(windows): Revisit cpu shares for windows (https://github.com/containerd/cri/issues/1297)
+const DefaultSandboxCPUshares = 2
+
+// WithRelativeRoot sets the root for the container
+func WithRelativeRoot(root string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		if s.Root == nil {
+			s.Root = &runtimespec.Root{}
+		}
+		s.Root.Path = root
+		return nil
+	}
+}
+
+// WithoutRoot sets the root to nil for the container.
+func WithoutRoot(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	s.Root = nil
+	return nil
+}
+
+// WithProcessArgs sets the process args on the spec based on the image and runtime config
+func WithProcessArgs(config *runtime.ContainerConfig, image *imagespec.ImageConfig) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		command, args := config.GetCommand(), config.GetArgs()
+		// The following logic is migrated from https://github.com/moby/moby/blob/master/daemon/commit.go
+		// TODO(random-liu): Clearly define the commands overwrite behavior.
+		if len(command) == 0 {
+			// Copy array to avoid data race.
+			if len(args) == 0 {
+				args = append([]string{}, image.Cmd...)
+			}
+			if command == nil {
+				command = append([]string{}, image.Entrypoint...)
+			}
+		}
+		if len(command) == 0 && len(args) == 0 {
+			return errors.New("no command specified")
+		}
+		return oci.WithProcessArgs(append(command, args...)...)(ctx, client, c, s)
+	}
+}
+
+// mounts defines how to sort runtime.Mount.
+// This is the same with the Docker implementation:
+//   https://github.com/moby/moby/blob/17.05.x/daemon/volumes.go#L26
+type orderedMounts []*runtime.Mount
+
+// Len returns the number of mounts. Used in sorting.
+func (m orderedMounts) Len() int {
+	return len(m)
+}
+
+// Less returns true if the number of parts (a/b/c would be 3 parts) in the
+// mount indexed by parameter 1 is less than that of the mount indexed by
+// parameter 2. Used in sorting.
+func (m orderedMounts) Less(i, j int) bool {
+	return m.parts(i) < m.parts(j)
+}
+
+// Swap swaps two items in an array of mounts. Used in sorting
+func (m orderedMounts) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+// parts returns the number of parts in the destination of a mount. Used in sorting.
+func (m orderedMounts) parts(i int) int {
+	return strings.Count(filepath.Clean(m[i].ContainerPath), string(os.PathSeparator))
+}
+
+// WithAnnotation sets the provided annotation
+func WithAnnotation(k, v string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Annotations == nil {
+			s.Annotations = make(map[string]string)
+		}
+		s.Annotations[k] = v
+		return nil
+	}
+}

pkg/containerd/opts/spec_linux.go

@@ -0,0 +1,719 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/oci"
+	"github.com/opencontainers/runc/libcontainer/devices"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	osinterface "github.com/containerd/cri/pkg/os"
+	"github.com/containerd/cri/pkg/util"
+)
+
+// WithAdditionalGIDs adds any additional groups listed for a particular user in the
+// /etc/groups file of the image's root filesystem to the OCI spec's additionalGids array.
+func WithAdditionalGIDs(userstr string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		if s.Process == nil {
+			s.Process = &runtimespec.Process{}
+		}
+		gids := s.Process.User.AdditionalGids
+		if err := oci.WithAdditionalGIDs(userstr)(ctx, client, c, s); err != nil {
+			return err
+		}
+		// Merge existing gids and new gids.
+		s.Process.User.AdditionalGids = mergeGids(s.Process.User.AdditionalGids, gids)
+		return nil
+	}
+}
+
+func mergeGids(gids1, gids2 []uint32) []uint32 {
+	gidsMap := make(map[uint32]struct{})
+	for _, gid1 := range gids1 {
+		gidsMap[gid1] = struct{}{}
+	}
+	for _, gid2 := range gids2 {
+		gidsMap[gid2] = struct{}{}
+	}
+	var gids []uint32
+	for gid := range gidsMap {
+		gids = append(gids, gid)
+	}
+	sort.Slice(gids, func(i, j int) bool { return gids[i] < gids[j] })
+	return gids
+}
+
+// WithoutRunMount removes the `/run` inside the spec
+func WithoutRunMount(_ context.Context, _ oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	var (
+		mounts  []runtimespec.Mount
+		current = s.Mounts
+	)
+	for _, m := range current {
+		if filepath.Clean(m.Destination) == "/run" {
+			continue
+		}
+		mounts = append(mounts, m)
+	}
+	s.Mounts = mounts
+	return nil
+}
+
+// WithoutDefaultSecuritySettings removes the default security settings generated on a spec
+func WithoutDefaultSecuritySettings(_ context.Context, _ oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	if s.Process == nil {
+		s.Process = &runtimespec.Process{}
+	}
+	// Make sure no default seccomp/apparmor is specified
+	s.Process.ApparmorProfile = ""
+	if s.Linux != nil {
+		s.Linux.Seccomp = nil
+	}
+	// Remove default rlimits (See issue #515)
+	s.Process.Rlimits = nil
+	return nil
+}
+
+// WithMounts sorts and adds runtime and CRI mounts to the spec
+func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, _ *containers.Container, s *runtimespec.Spec) (err error) {
+		// mergeMounts merge CRI mounts with extra mounts. If a mount destination
+		// is mounted by both a CRI mount and an extra mount, the CRI mount will
+		// be kept.
+		var (
+			criMounts = config.GetMounts()
+			mounts    = append([]*runtime.Mount{}, criMounts...)
+		)
+		// Copy all mounts from extra mounts, except for mounts overridden by CRI.
+		for _, e := range extra {
+			found := false
+			for _, c := range criMounts {
+				if filepath.Clean(e.ContainerPath) == filepath.Clean(c.ContainerPath) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				mounts = append(mounts, e)
+			}
+		}
+
+		// Sort mounts in number of parts. This ensures that high level mounts don't
+		// shadow other mounts.
+		sort.Sort(orderedMounts(mounts))
+
+		// Mount cgroup into the container as readonly, which inherits docker's behavior.
+		s.Mounts = append(s.Mounts, runtimespec.Mount{
+			Source:      "cgroup",
+			Destination: "/sys/fs/cgroup",
+			Type:        "cgroup",
+			Options:     []string{"nosuid", "noexec", "nodev", "relatime", "ro"},
+		})
+
+		// Copy all mounts from default mounts, except for
+		// - mounts overridden by supplied mount;
+		// - all mounts under /dev if a supplied /dev is present.
+		mountSet := make(map[string]struct{})
+		for _, m := range mounts {
+			mountSet[filepath.Clean(m.ContainerPath)] = struct{}{}
+		}
+
+		defaultMounts := s.Mounts
+		s.Mounts = nil
+
+		for _, m := range defaultMounts {
+			dst := filepath.Clean(m.Destination)
+			if _, ok := mountSet[dst]; ok {
+				// filter out mount overridden by a supplied mount
+				continue
+			}
+			if _, mountDev := mountSet["/dev"]; mountDev && strings.HasPrefix(dst, "/dev/") {
+				// filter out everything under /dev if /dev is a supplied mount
+				continue
+			}
+			s.Mounts = append(s.Mounts, m)
+		}
+
+		for _, mount := range mounts {
+			var (
+				dst = mount.GetContainerPath()
+				src = mount.GetHostPath()
+			)
+			// Create the host path if it doesn't exist.
+			// TODO(random-liu): Add CRI validation test for this case.
+			if _, err := osi.Stat(src); err != nil {
+				if !os.IsNotExist(err) {
+					return errors.Wrapf(err, "failed to stat %q", src)
+				}
+				if err := osi.MkdirAll(src, 0755); err != nil {
+					return errors.Wrapf(err, "failed to mkdir %q", src)
+				}
+			}
+			// TODO(random-liu): Add cri-containerd integration test or cri validation test
+			// for this.
+			src, err := osi.ResolveSymbolicLink(src)
+			if err != nil {
+				return errors.Wrapf(err, "failed to resolve symlink %q", src)
+			}
+			if s.Linux == nil {
+				s.Linux = &runtimespec.Linux{}
+			}
+			options := []string{"rbind"}
+			switch mount.GetPropagation() {
+			case runtime.MountPropagation_PROPAGATION_PRIVATE:
+				options = append(options, "rprivate")
+				// Since default root propagation in runc is rprivate ignore
+				// setting the root propagation
+			case runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL:
+				if err := ensureShared(src, osi.(osinterface.UNIX).LookupMount); err != nil {
+					return err
+				}
+				options = append(options, "rshared")
+				s.Linux.RootfsPropagation = "rshared"
+			case runtime.MountPropagation_PROPAGATION_HOST_TO_CONTAINER:
+				if err := ensureSharedOrSlave(src, osi.(osinterface.UNIX).LookupMount); err != nil {
+					return err
+				}
+				options = append(options, "rslave")
+				if s.Linux.RootfsPropagation != "rshared" &&
+					s.Linux.RootfsPropagation != "rslave" {
+					s.Linux.RootfsPropagation = "rslave"
+				}
+			default:
+				log.G(ctx).Warnf("Unknown propagation mode for hostPath %q", mount.HostPath)
+				options = append(options, "rprivate")
+			}
+
+			// NOTE(random-liu): we don't change all mounts to `ro` when root filesystem
+			// is readonly. This is different from docker's behavior, but make more sense.
+			if mount.GetReadonly() {
+				options = append(options, "ro")
+			} else {
+				options = append(options, "rw")
+			}
+
+			if mount.GetSelinuxRelabel() {
+				if err := label.Relabel(src, mountLabel, false); err != nil && err != unix.ENOTSUP {
+					return errors.Wrapf(err, "relabel %q with %q failed", src, mountLabel)
+				}
+			}
+			s.Mounts = append(s.Mounts, runtimespec.Mount{
+				Source:      src,
+				Destination: dst,
+				Type:        "bind",
+				Options:     options,
+			})
+		}
+		return nil
+	}
+}
+
+// Ensure mount point on which path is mounted, is shared.
+func ensureShared(path string, lookupMount func(string) (mount.Info, error)) error {
+	mountInfo, err := lookupMount(path)
+	if err != nil {
+		return err
+	}
+
+	// Make sure source mount point is shared.
+	optsSplit := strings.Split(mountInfo.Optional, " ")
+	for _, opt := range optsSplit {
+		if strings.HasPrefix(opt, "shared:") {
+			return nil
+		}
+	}
+
+	return errors.Errorf("path %q is mounted on %q but it is not a shared mount", path, mountInfo.Mountpoint)
+}
+
+// ensure mount point on which path is mounted, is either shared or slave.
+func ensureSharedOrSlave(path string, lookupMount func(string) (mount.Info, error)) error {
+	mountInfo, err := lookupMount(path)
+	if err != nil {
+		return err
+	}
+	// Make sure source mount point is shared.
+	optsSplit := strings.Split(mountInfo.Optional, " ")
+	for _, opt := range optsSplit {
+		if strings.HasPrefix(opt, "shared:") {
+			return nil
+		} else if strings.HasPrefix(opt, "master:") {
+			return nil
+		}
+	}
+	return errors.Errorf("path %q is mounted on %q but it is not a shared or slave mount", path, mountInfo.Mountpoint)
+}
+
+func addDevice(s *runtimespec.Spec, rd runtimespec.LinuxDevice) {
+	for i, dev := range s.Linux.Devices {
+		if dev.Path == rd.Path {
+			s.Linux.Devices[i] = rd
+			return
+		}
+	}
+	s.Linux.Devices = append(s.Linux.Devices, rd)
+}
+
+// WithDevices sets the provided devices onto the container spec
+func WithDevices(osi osinterface.OS, config *runtime.ContainerConfig) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		if s.Linux == nil {
+			s.Linux = &runtimespec.Linux{}
+		}
+		if s.Linux.Resources == nil {
+			s.Linux.Resources = &runtimespec.LinuxResources{}
+		}
+		for _, device := range config.GetDevices() {
+			path, err := osi.ResolveSymbolicLink(device.HostPath)
+			if err != nil {
+				return err
+			}
+			dev, err := devices.DeviceFromPath(path, device.Permissions)
+			if err != nil {
+				return err
+			}
+			rd := runtimespec.LinuxDevice{
+				Path:  device.ContainerPath,
+				Type:  string(dev.Type),
+				Major: dev.Major,
+				Minor: dev.Minor,
+				UID:   &dev.Uid,
+				GID:   &dev.Gid,
+			}
+
+			addDevice(s, rd)
+
+			s.Linux.Resources.Devices = append(s.Linux.Resources.Devices, runtimespec.LinuxDeviceCgroup{
+				Allow:  true,
+				Type:   string(dev.Type),
+				Major:  &dev.Major,
+				Minor:  &dev.Minor,
+				Access: string(dev.Permissions),
+			})
+		}
+		return nil
+	}
+}
+
+// WithCapabilities sets the provided capabilties from the security context
+func WithCapabilities(sc *runtime.LinuxContainerSecurityContext) oci.SpecOpts {
+	capabilities := sc.GetCapabilities()
+	if capabilities == nil {
+		return nullOpt
+	}
+
+	var opts []oci.SpecOpts
+	// Add/drop all capabilities if "all" is specified, so that
+	// following individual add/drop could still work. E.g.
+	// AddCapabilities: []string{"ALL"}, DropCapabilities: []string{"CHOWN"}
+	// will be all capabilities without `CAP_CHOWN`.
+	if util.InStringSlice(capabilities.GetAddCapabilities(), "ALL") {
+		opts = append(opts, oci.WithAllCapabilities)
+	}
+	if util.InStringSlice(capabilities.GetDropCapabilities(), "ALL") {
+		opts = append(opts, oci.WithCapabilities(nil))
+	}
+
+	var caps []string
+	for _, c := range capabilities.GetAddCapabilities() {
+		if strings.ToUpper(c) == "ALL" {
+			continue
+		}
+		// Capabilities in CRI doesn't have `CAP_` prefix, so add it.
+		caps = append(caps, "CAP_"+strings.ToUpper(c))
+	}
+	opts = append(opts, oci.WithAddedCapabilities(caps))
+
+	caps = []string{}
+	for _, c := range capabilities.GetDropCapabilities() {
+		if strings.ToUpper(c) == "ALL" {
+			continue
+		}
+		caps = append(caps, "CAP_"+strings.ToUpper(c))
+	}
+	opts = append(opts, oci.WithDroppedCapabilities(caps))
+	return oci.Compose(opts...)
+}
+
+// WithoutAmbientCaps removes the ambient caps from the spec
+func WithoutAmbientCaps(_ context.Context, _ oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	if s.Process == nil {
+		s.Process = &runtimespec.Process{}
+	}
+	if s.Process.Capabilities == nil {
+		s.Process.Capabilities = &runtimespec.LinuxCapabilities{}
+	}
+	s.Process.Capabilities.Ambient = nil
+	return nil
+}
+
+// WithDisabledCgroups clears the Cgroups Path from the spec
+func WithDisabledCgroups(_ context.Context, _ oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	if s.Linux == nil {
+		s.Linux = &runtimespec.Linux{}
+	}
+	s.Linux.CgroupsPath = ""
+	return nil
+}
+
+// WithSelinuxLabels sets the mount and process labels
+func WithSelinuxLabels(process, mount string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		if s.Linux == nil {
+			s.Linux = &runtimespec.Linux{}
+		}
+		if s.Process == nil {
+			s.Process = &runtimespec.Process{}
+		}
+		s.Linux.MountLabel = mount
+		s.Process.SelinuxLabel = process
+		return nil
+	}
+}
+
+// WithResources sets the provided resource restrictions
+func WithResources(resources *runtime.LinuxContainerResources, tolerateMissingHugetlbController, disableHugetlbController bool) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) (err error) {
+		if resources == nil {
+			return nil
+		}
+		if s.Linux == nil {
+			s.Linux = &runtimespec.Linux{}
+		}
+		if s.Linux.Resources == nil {
+			s.Linux.Resources = &runtimespec.LinuxResources{}
+		}
+		if s.Linux.Resources.CPU == nil {
+			s.Linux.Resources.CPU = &runtimespec.LinuxCPU{}
+		}
+		if s.Linux.Resources.Memory == nil {
+			s.Linux.Resources.Memory = &runtimespec.LinuxMemory{}
+		}
+		var (
+			p         = uint64(resources.GetCpuPeriod())
+			q         = resources.GetCpuQuota()
+			shares    = uint64(resources.GetCpuShares())
+			limit     = resources.GetMemoryLimitInBytes()
+			hugepages = resources.GetHugepageLimits()
+		)
+
+		if p != 0 {
+			s.Linux.Resources.CPU.Period = &p
+		}
+		if q != 0 {
+			s.Linux.Resources.CPU.Quota = &q
+		}
+		if shares != 0 {
+			s.Linux.Resources.CPU.Shares = &shares
+		}
+		if cpus := resources.GetCpusetCpus(); cpus != "" {
+			s.Linux.Resources.CPU.Cpus = cpus
+		}
+		if mems := resources.GetCpusetMems(); mems != "" {
+			s.Linux.Resources.CPU.Mems = resources.GetCpusetMems()
+		}
+		if limit != 0 {
+			s.Linux.Resources.Memory.Limit = &limit
+		}
+		if !disableHugetlbController {
+			if isHugetlbControllerPresent() {
+				for _, limit := range hugepages {
+					s.Linux.Resources.HugepageLimits = append(s.Linux.Resources.HugepageLimits, runtimespec.LinuxHugepageLimit{
+						Pagesize: limit.PageSize,
+						Limit:    limit.Limit,
+					})
+				}
+			} else {
+				if !tolerateMissingHugetlbController {
+					return errors.Errorf("huge pages limits are specified but hugetlb cgroup controller is missing. " +
+						"Please set tolerate_missing_hugetlb_controller to `true` to ignore this error")
+				}
+				logrus.Warn("hugetlb cgroup controller is absent. skipping huge pages limits")
+			}
+		}
+		return nil
+	}
+}
+
+var (
+	supportsHugetlbOnce sync.Once
+	supportsHugetlb     bool
+)
+
+func isHugetlbControllerPresent() bool {
+	supportsHugetlbOnce.Do(func() {
+		supportsHugetlb = false
+		if IsCgroup2UnifiedMode() {
+			supportsHugetlb, _ = cgroupv2HasHugetlb()
+		} else {
+			supportsHugetlb, _ = cgroupv1HasHugetlb()
+		}
+	})
+	return supportsHugetlb
+}
+
+var (
+	_cgroupv1HasHugetlbOnce sync.Once
+	_cgroupv1HasHugetlb     bool
+	_cgroupv1HasHugetlbErr  error
+	_cgroupv2HasHugetlbOnce sync.Once
+	_cgroupv2HasHugetlb     bool
+	_cgroupv2HasHugetlbErr  error
+	isUnifiedOnce           sync.Once
+	isUnified               bool
+)
+
+// cgroupv1HasHugetlb returns whether the hugetlb controller is present on
+// cgroup v1.
+func cgroupv1HasHugetlb() (bool, error) {
+	_cgroupv1HasHugetlbOnce.Do(func() {
+		if _, err := ioutil.ReadDir("/sys/fs/cgroup/hugetlb"); err != nil {
+			_cgroupv1HasHugetlbErr = errors.Wrap(err, "readdir /sys/fs/cgroup/hugetlb")
+			_cgroupv1HasHugetlb = false
+		} else {
+			_cgroupv1HasHugetlbErr = nil
+			_cgroupv1HasHugetlb = true
+		}
+	})
+	return _cgroupv1HasHugetlb, _cgroupv1HasHugetlbErr
+}
+
+// cgroupv2HasHugetlb returns whether the hugetlb controller is present on
+// cgroup v2.
+func cgroupv2HasHugetlb() (bool, error) {
+	_cgroupv2HasHugetlbOnce.Do(func() {
+		controllers, err := ioutil.ReadFile("/sys/fs/cgroup/cgroup.controllers")
+		if err != nil {
+			_cgroupv2HasHugetlbErr = errors.Wrap(err, "read /sys/fs/cgroup/cgroup.controllers")
+			return
+		}
+		_cgroupv2HasHugetlb = strings.Contains(string(controllers), "hugetlb")
+	})
+	return _cgroupv2HasHugetlb, _cgroupv2HasHugetlbErr
+}
+
+// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
+func IsCgroup2UnifiedMode() bool {
+	isUnifiedOnce.Do(func() {
+		var st syscall.Statfs_t
+		if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
+			panic("cannot statfs cgroup root")
+		}
+		isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC
+	})
+	return isUnified
+}
+
+// WithOOMScoreAdj sets the oom score
+func WithOOMScoreAdj(config *runtime.ContainerConfig, restrict bool) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Process == nil {
+			s.Process = &runtimespec.Process{}
+		}
+
+		resources := config.GetLinux().GetResources()
+		if resources == nil {
+			return nil
+		}
+		adj := int(resources.GetOomScoreAdj())
+		if restrict {
+			var err error
+			adj, err = restrictOOMScoreAdj(adj)
+			if err != nil {
+				return err
+			}
+		}
+		s.Process.OOMScoreAdj = &adj
+		return nil
+	}
+}
+
+// WithSysctls sets the provided sysctls onto the spec
+func WithSysctls(sysctls map[string]string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Linux == nil {
+			s.Linux = &runtimespec.Linux{}
+		}
+		if s.Linux.Sysctl == nil {
+			s.Linux.Sysctl = make(map[string]string)
+		}
+		for k, v := range sysctls {
+			s.Linux.Sysctl[k] = v
+		}
+		return nil
+	}
+}
+
+// WithPodOOMScoreAdj sets the oom score for the pod sandbox
+func WithPodOOMScoreAdj(adj int, restrict bool) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Process == nil {
+			s.Process = &runtimespec.Process{}
+		}
+		if restrict {
+			var err error
+			adj, err = restrictOOMScoreAdj(adj)
+			if err != nil {
+				return err
+			}
+		}
+		s.Process.OOMScoreAdj = &adj
+		return nil
+	}
+}
+
+// WithSupplementalGroups sets the supplemental groups for the process
+func WithSupplementalGroups(groups []int64) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Process == nil {
+			s.Process = &runtimespec.Process{}
+		}
+		var guids []uint32
+		for _, g := range groups {
+			guids = append(guids, uint32(g))
+		}
+		s.Process.User.AdditionalGids = mergeGids(s.Process.User.AdditionalGids, guids)
+		return nil
+	}
+}
+
+// WithPodNamespaces sets the pod namespaces for the container
+func WithPodNamespaces(config *runtime.LinuxContainerSecurityContext, pid uint32) oci.SpecOpts {
+	namespaces := config.GetNamespaceOptions()
+
+	opts := []oci.SpecOpts{
+		oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.NetworkNamespace, Path: GetNetworkNamespace(pid)}),
+		oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.IPCNamespace, Path: GetIPCNamespace(pid)}),
+		oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.UTSNamespace, Path: GetUTSNamespace(pid)}),
+	}
+	if namespaces.GetPid() != runtime.NamespaceMode_CONTAINER {
+		opts = append(opts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{Type: runtimespec.PIDNamespace, Path: GetPIDNamespace(pid)}))
+	}
+	return oci.Compose(opts...)
+}
+
+// WithDefaultSandboxShares sets the default sandbox CPU shares
+func WithDefaultSandboxShares(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	if s.Linux == nil {
+		s.Linux = &runtimespec.Linux{}
+	}
+	if s.Linux.Resources == nil {
+		s.Linux.Resources = &runtimespec.LinuxResources{}
+	}
+	if s.Linux.Resources.CPU == nil {
+		s.Linux.Resources.CPU = &runtimespec.LinuxCPU{}
+	}
+	i := uint64(DefaultSandboxCPUshares)
+	s.Linux.Resources.CPU.Shares = &i
+	return nil
+}
+
+// WithoutNamespace removes the provided namespace
+func WithoutNamespace(t runtimespec.LinuxNamespaceType) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Linux == nil {
+			return nil
+		}
+		var namespaces []runtimespec.LinuxNamespace
+		for i, ns := range s.Linux.Namespaces {
+			if ns.Type != t {
+				namespaces = append(namespaces, s.Linux.Namespaces[i])
+			}
+		}
+		s.Linux.Namespaces = namespaces
+		return nil
+	}
+}
+
+func nullOpt(_ context.Context, _ oci.Client, _ *containers.Container, _ *runtimespec.Spec) error {
+	return nil
+}
+
+func getCurrentOOMScoreAdj() (int, error) {
+	b, err := ioutil.ReadFile("/proc/self/oom_score_adj")
+	if err != nil {
+		return 0, errors.Wrap(err, "could not get the daemon oom_score_adj")
+	}
+	s := strings.TrimSpace(string(b))
+	i, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, errors.Wrap(err, "could not get the daemon oom_score_adj")
+	}
+	return i, nil
+}
+
+func restrictOOMScoreAdj(preferredOOMScoreAdj int) (int, error) {
+	currentOOMScoreAdj, err := getCurrentOOMScoreAdj()
+	if err != nil {
+		return preferredOOMScoreAdj, err
+	}
+	if preferredOOMScoreAdj < currentOOMScoreAdj {
+		return currentOOMScoreAdj, nil
+	}
+	return preferredOOMScoreAdj, nil
+}
+
+const (
+	// netNSFormat is the format of network namespace of a process.
+	netNSFormat = "/proc/%v/ns/net"
+	// ipcNSFormat is the format of ipc namespace of a process.
+	ipcNSFormat = "/proc/%v/ns/ipc"
+	// utsNSFormat is the format of uts namespace of a process.
+	utsNSFormat = "/proc/%v/ns/uts"
+	// pidNSFormat is the format of pid namespace of a process.
+	pidNSFormat = "/proc/%v/ns/pid"
+)
+
+// GetNetworkNamespace returns the network namespace of a process.
+func GetNetworkNamespace(pid uint32) string {
+	return fmt.Sprintf(netNSFormat, pid)
+}
+
+// GetIPCNamespace returns the ipc namespace of a process.
+func GetIPCNamespace(pid uint32) string {
+	return fmt.Sprintf(ipcNSFormat, pid)
+}
+
+// GetUTSNamespace returns the uts namespace of a process.
+func GetUTSNamespace(pid uint32) string {
+	return fmt.Sprintf(utsNSFormat, pid)
+}
+
+// GetPIDNamespace returns the pid namespace of a process.
+func GetPIDNamespace(pid uint32) string {
+	return fmt.Sprintf(pidNSFormat, pid)
+}

pkg/containerd/opts/spec_linux_test.go

@@ -0,0 +1,47 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeGids(t *testing.T) {
+	gids1 := []uint32{3, 2, 1}
+	gids2 := []uint32{2, 3, 4}
+	assert.Equal(t, []uint32{1, 2, 3, 4}, mergeGids(gids1, gids2))
+}
+
+func TestRestrictOOMScoreAdj(t *testing.T) {
+	current, err := getCurrentOOMScoreAdj()
+	require.NoError(t, err)
+
+	got, err := restrictOOMScoreAdj(current - 1)
+	require.NoError(t, err)
+	assert.Equal(t, got, current)
+
+	got, err = restrictOOMScoreAdj(current)
+	require.NoError(t, err)
+	assert.Equal(t, got, current)
+
+	got, err = restrictOOMScoreAdj(current + 1)
+	require.NoError(t, err)
+	assert.Equal(t, got, current+1)
+}

pkg/containerd/opts/spec_test.go

@@ -0,0 +1,46 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestOrderedMounts(t *testing.T) {
+	mounts := []*runtime.Mount{
+		{ContainerPath: "/a/b/c"},
+		{ContainerPath: "/a/b"},
+		{ContainerPath: "/a/b/c/d"},
+		{ContainerPath: "/a"},
+		{ContainerPath: "/b"},
+		{ContainerPath: "/b/c"},
+	}
+	expected := []*runtime.Mount{
+		{ContainerPath: "/a"},
+		{ContainerPath: "/b"},
+		{ContainerPath: "/a/b"},
+		{ContainerPath: "/b/c"},
+		{ContainerPath: "/a/b/c"},
+		{ContainerPath: "/a/b/c/d"},
+	}
+	sort.Stable(orderedMounts(mounts))
+	assert.Equal(t, expected, mounts)
+}

pkg/containerd/opts/spec_windows.go

@@ -0,0 +1,224 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"context"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	osinterface "github.com/containerd/cri/pkg/os"
+)
+
+// WithWindowsNetworkNamespace sets windows network namespace for container.
+// TODO(windows): Move this into container/containerd.
+func WithWindowsNetworkNamespace(path string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Windows == nil {
+			s.Windows = &runtimespec.Windows{}
+		}
+		if s.Windows.Network == nil {
+			s.Windows.Network = &runtimespec.WindowsNetwork{}
+		}
+		s.Windows.Network.NetworkNamespace = path
+		return nil
+	}
+}
+
+// namedPipePath returns true if the given path is to a named pipe.
+func namedPipePath(p string) bool {
+	return strings.HasPrefix(p, `\\.\pipe\`)
+}
+
+// cleanMount returns a cleaned version of the mount path. The input is returned
+// as-is if it is a named pipe path.
+func cleanMount(p string) string {
+	if namedPipePath(p) {
+		return p
+	}
+	return filepath.Clean(p)
+}
+
+// WithWindowsMounts sorts and adds runtime and CRI mounts to the spec for
+// windows container.
+func WithWindowsMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, _ *containers.Container, s *runtimespec.Spec) error {
+		// mergeMounts merge CRI mounts with extra mounts. If a mount destination
+		// is mounted by both a CRI mount and an extra mount, the CRI mount will
+		// be kept.
+		var (
+			criMounts = config.GetMounts()
+			mounts    = append([]*runtime.Mount{}, criMounts...)
+		)
+		// Copy all mounts from extra mounts, except for mounts overridden by CRI.
+		for _, e := range extra {
+			found := false
+			for _, c := range criMounts {
+				if cleanMount(e.ContainerPath) == cleanMount(c.ContainerPath) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				mounts = append(mounts, e)
+			}
+		}
+
+		// Sort mounts in number of parts. This ensures that high level mounts don't
+		// shadow other mounts.
+		sort.Sort(orderedMounts(mounts))
+
+		// Copy all mounts from default mounts, except for
+		// mounts overridden by supplied mount;
+		mountSet := make(map[string]struct{})
+		for _, m := range mounts {
+			mountSet[cleanMount(m.ContainerPath)] = struct{}{}
+		}
+
+		defaultMounts := s.Mounts
+		s.Mounts = nil
+
+		for _, m := range defaultMounts {
+			dst := cleanMount(m.Destination)
+			if _, ok := mountSet[dst]; ok {
+				// filter out mount overridden by a supplied mount
+				continue
+			}
+			s.Mounts = append(s.Mounts, m)
+		}
+
+		for _, mount := range mounts {
+			var (
+				dst = mount.GetContainerPath()
+				src = mount.GetHostPath()
+			)
+			// In the case of a named pipe mount on Windows, don't stat the file
+			// or do other operations that open it, as that could interfere with
+			// the listening process. filepath.Clean also breaks named pipe
+			// paths, so don't use it.
+			if !namedPipePath(src) {
+				if _, err := osi.Stat(src); err != nil {
+					// If the source doesn't exist, return an error instead
+					// of creating the source. This aligns with Docker's
+					// behavior on windows.
+					return errors.Wrapf(err, "failed to stat %q", src)
+				}
+				var err error
+				src, err = osi.ResolveSymbolicLink(src)
+				if err != nil {
+					return errors.Wrapf(err, "failed to resolve symlink %q", src)
+				}
+				// hcsshim requires clean path, especially '/' -> '\'.
+				src = filepath.Clean(src)
+				dst = filepath.Clean(dst)
+			}
+
+			var options []string
+			// NOTE(random-liu): we don't change all mounts to `ro` when root filesystem
+			// is readonly. This is different from docker's behavior, but make more sense.
+			if mount.GetReadonly() {
+				options = append(options, "ro")
+			} else {
+				options = append(options, "rw")
+			}
+			s.Mounts = append(s.Mounts, runtimespec.Mount{
+				Source:      src,
+				Destination: dst,
+				Options:     options,
+			})
+		}
+		return nil
+	}
+}
+
+// WithWindowsResources sets the provided resource restrictions for windows.
+func WithWindowsResources(resources *runtime.WindowsContainerResources) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if resources == nil {
+			return nil
+		}
+		if s.Windows == nil {
+			s.Windows = &runtimespec.Windows{}
+		}
+		if s.Windows.Resources == nil {
+			s.Windows.Resources = &runtimespec.WindowsResources{}
+		}
+		if s.Windows.Resources.CPU == nil {
+			s.Windows.Resources.CPU = &runtimespec.WindowsCPUResources{}
+		}
+		if s.Windows.Resources.Memory == nil {
+			s.Windows.Resources.Memory = &runtimespec.WindowsMemoryResources{}
+		}
+
+		var (
+			count  = uint64(resources.GetCpuCount())
+			shares = uint16(resources.GetCpuShares())
+			max    = uint16(resources.GetCpuMaximum())
+			limit  = uint64(resources.GetMemoryLimitInBytes())
+		)
+		if count != 0 {
+			s.Windows.Resources.CPU.Count = &count
+		}
+		if shares != 0 {
+			s.Windows.Resources.CPU.Shares = &shares
+		}
+		if max != 0 {
+			s.Windows.Resources.CPU.Maximum = &max
+		}
+		if limit != 0 {
+			s.Windows.Resources.Memory.Limit = &limit
+		}
+		return nil
+	}
+}
+
+// WithWindowsDefaultSandboxShares sets the default sandbox CPU shares
+func WithWindowsDefaultSandboxShares(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+	if s.Windows == nil {
+		s.Windows = &runtimespec.Windows{}
+	}
+	if s.Windows.Resources == nil {
+		s.Windows.Resources = &runtimespec.WindowsResources{}
+	}
+	if s.Windows.Resources.CPU == nil {
+		s.Windows.Resources.CPU = &runtimespec.WindowsCPUResources{}
+	}
+	i := uint16(DefaultSandboxCPUshares)
+	s.Windows.Resources.CPU.Shares = &i
+	return nil
+}
+
+// WithWindowsCredentialSpec assigns `credentialSpec` to the
+// `runtime.Spec.Windows.CredentialSpec` field.
+func WithWindowsCredentialSpec(credentialSpec string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *runtimespec.Spec) error {
+		if s.Windows == nil {
+			s.Windows = &runtimespec.Windows{}
+		}
+		s.Windows.CredentialSpec = credentialSpec
+		return nil
+	}
+}

pkg/containerd/opts/task.go

@@ -0,0 +1,38 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package opts
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+)
+
+// WithContainerdShimCgroup returns function that sets the containerd
+// shim cgroup path
+func WithContainerdShimCgroup(path string) containerd.NewTaskOpts {
+	return func(_ context.Context, _ *containerd.Client, r *containerd.TaskInfo) error {
+		r.Options = &runctypes.CreateOptions{
+			ShimCgroup: path,
+		}
+		return nil
+	}
+}
+
+//TODO: Since Options is an interface different WithXXX will be needed to set different
+// combinations of CreateOptions.

pkg/containerd/platforms/default_unix.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package platforms
+
+import (
+	"github.com/containerd/containerd/platforms"
+)
+
+// Default returns the current platform's default platform specification.
+func Default() platforms.MatchComparer {
+	return platforms.Default()
+}

pkg/containerd/platforms/default_windows.go

@@ -0,0 +1,77 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package platforms
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/containerd/platforms"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/sys/windows"
+)
+
+type matchComparer struct {
+	defaults        platforms.Matcher
+	osVersionPrefix string
+}
+
+// Match matches platform with the same windows major, minor
+// and build version.
+func (m matchComparer) Match(p imagespec.Platform) bool {
+	if m.defaults.Match(p) {
+		// TODO(windows): Figure out whether OSVersion is deprecated.
+		return strings.HasPrefix(p.OSVersion, m.osVersionPrefix)
+	}
+	return false
+}
+
+// Less sorts matched platforms in front of other platforms.
+// For matched platforms, it puts platforms with larger revision
+// number in front.
+func (m matchComparer) Less(p1, p2 imagespec.Platform) bool {
+	m1, m2 := m.Match(p1), m.Match(p2)
+	if m1 && m2 {
+		r1, r2 := revision(p1.OSVersion), revision(p2.OSVersion)
+		return r1 > r2
+	}
+	return m1 && !m2
+}
+
+func revision(v string) int {
+	parts := strings.Split(v, ".")
+	if len(parts) < 4 {
+		return 0
+	}
+	r, err := strconv.Atoi(parts[3])
+	if err != nil {
+		return 0
+	}
+	return r
+}
+
+// Default returns the current platform's default platform specification.
+func Default() platforms.MatchComparer {
+	major, minor, build := windows.RtlGetNtVersionNumbers()
+	return matchComparer{
+		defaults:        platforms.Only(platforms.DefaultSpec()),
+		osVersionPrefix: fmt.Sprintf("%d.%d.%d", major, minor, build),
+	}
+}

pkg/containerd/platforms/default_windows_test.go

@@ -0,0 +1,150 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package platforms
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/containerd/containerd/platforms"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMatchComparerMatch(t *testing.T) {
+	m := matchComparer{
+		defaults: platforms.Only(imagespec.Platform{
+			Architecture: "amd64",
+			OS:           "windows",
+		}),
+		osVersionPrefix: "10.0.17763",
+	}
+	for _, test := range []struct {
+		platform imagespec.Platform
+		match    bool
+	}{
+		{
+			platform: imagespec.Platform{
+				Architecture: "amd64",
+				OS:           "windows",
+				OSVersion:    "10.0.17763.1",
+			},
+			match: true,
+		},
+		{
+			platform: imagespec.Platform{
+				Architecture: "amd64",
+				OS:           "windows",
+				OSVersion:    "10.0.17763.2",
+			},
+			match: true,
+		},
+		{
+			platform: imagespec.Platform{
+				Architecture: "amd64",
+				OS:           "windows",
+				OSVersion:    "10.0.17762.1",
+			},
+			match: false,
+		},
+		{
+			platform: imagespec.Platform{
+				Architecture: "amd64",
+				OS:           "windows",
+				OSVersion:    "10.0.17764.1",
+			},
+			match: false,
+		},
+		{
+			platform: imagespec.Platform{
+				Architecture: "amd64",
+				OS:           "windows",
+			},
+			match: false,
+		},
+	} {
+		assert.Equal(t, test.match, m.Match(test.platform))
+	}
+}
+
+func TestMatchComparerLess(t *testing.T) {
+	m := matchComparer{
+		defaults: platforms.Only(imagespec.Platform{
+			Architecture: "amd64",
+			OS:           "windows",
+		}),
+		osVersionPrefix: "10.0.17763",
+	}
+	platforms := []imagespec.Platform{
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17764.1",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17763.1",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17763.2",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17762.1",
+		},
+	}
+	expected := []imagespec.Platform{
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17763.2",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17763.1",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17764.1",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+		},
+		{
+			Architecture: "amd64",
+			OS:           "windows",
+			OSVersion:    "10.0.17762.1",
+		},
+	}
+	sort.SliceStable(platforms, func(i, j int) bool {
+		return m.Less(platforms[i], platforms[j])
+	})
+	assert.Equal(t, expected, platforms)
+}

pkg/containerd/util/util.go

@@ -0,0 +1,46 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package util
+
+import (
+	"time"
+
+	"github.com/containerd/containerd/namespaces"
+	"golang.org/x/net/context"
+
+	"github.com/containerd/cri/pkg/constants"
+)
+
+// deferCleanupTimeout is the default timeout for containerd cleanup operations
+// in defer.
+const deferCleanupTimeout = 1 * time.Minute
+
+// DeferContext returns a context for containerd cleanup operations in defer.
+// A default timeout is applied to avoid cleanup operation pending forever.
+func DeferContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(NamespacedContext(), deferCleanupTimeout)
+}
+
+// NamespacedContext returns a context with kubernetes namespace set.
+func NamespacedContext() context.Context {
+	return WithNamespace(context.Background())
+}
+
+// WithNamespace adds kubernetes namespace to the context.
+func WithNamespace(ctx context.Context) context.Context {
+	return namespaces.WithNamespace(ctx, constants.K8sContainerdNamespace)
+}

pkg/cri/cri.go

@@ -0,0 +1,192 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cri
+
+import (
+	"flag"
+	"path/filepath"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/api/services/containers/v1"
+	"github.com/containerd/containerd/api/services/diff/v1"
+	"github.com/containerd/containerd/api/services/images/v1"
+	introspectionapi "github.com/containerd/containerd/api/services/introspection/v1"
+	"github.com/containerd/containerd/api/services/namespaces/v1"
+	"github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/services"
+	"github.com/containerd/containerd/snapshots"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/klog/v2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/constants"
+	criplatforms "github.com/containerd/cri/pkg/containerd/platforms"
+	"github.com/containerd/cri/pkg/server"
+)
+
+// TODO(random-liu): Use github.com/pkg/errors for our errors.
+// Register CRI service plugin
+func init() {
+	config := criconfig.DefaultConfig()
+	plugin.Register(&plugin.Registration{
+		Type:   plugin.GRPCPlugin,
+		ID:     "cri",
+		Config: &config,
+		Requires: []plugin.Type{
+			plugin.ServicePlugin,
+		},
+		InitFn: initCRIService,
+	})
+}
+
+func initCRIService(ic *plugin.InitContext) (interface{}, error) {
+	ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()}
+	ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion}
+	ctx := ic.Context
+	pluginConfig := ic.Config.(*criconfig.PluginConfig)
+	if err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil {
+		return nil, errors.Wrap(err, "invalid plugin config")
+	}
+
+	c := criconfig.Config{
+		PluginConfig:       *pluginConfig,
+		ContainerdRootDir:  filepath.Dir(ic.Root),
+		ContainerdEndpoint: ic.Address,
+		RootDir:            ic.Root,
+		StateDir:           ic.State,
+	}
+	log.G(ctx).Infof("Start cri plugin with config %+v", c)
+
+	if err := setGLogLevel(); err != nil {
+		return nil, errors.Wrap(err, "failed to set glog level")
+	}
+
+	servicesOpts, err := getServicesOpts(ic)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get services")
+	}
+
+	log.G(ctx).Info("Connect containerd service")
+	client, err := containerd.New(
+		"",
+		containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
+		containerd.WithDefaultPlatform(criplatforms.Default()),
+		containerd.WithServices(servicesOpts...),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd client")
+	}
+
+	s, err := server.NewCRIService(c, client)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create CRI service")
+	}
+
+	go func() {
+		if err := s.Run(); err != nil {
+			log.G(ctx).WithError(err).Fatal("Failed to run CRI service")
+		}
+		// TODO(random-liu): Whether and how we can stop containerd.
+	}()
+	return s, nil
+}
+
+// getServicesOpts get service options from plugin context.
+func getServicesOpts(ic *plugin.InitContext) ([]containerd.ServicesOpt, error) {
+	plugins, err := ic.GetByType(plugin.ServicePlugin)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get service plugin")
+	}
+
+	opts := []containerd.ServicesOpt{
+		containerd.WithEventService(ic.Events),
+	}
+	for s, fn := range map[string]func(interface{}) containerd.ServicesOpt{
+		services.ContentService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithContentStore(s.(content.Store))
+		},
+		services.ImagesService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithImageService(s.(images.ImagesClient))
+		},
+		services.SnapshotsService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithSnapshotters(s.(map[string]snapshots.Snapshotter))
+		},
+		services.ContainersService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithContainerService(s.(containers.ContainersClient))
+		},
+		services.TasksService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithTaskService(s.(tasks.TasksClient))
+		},
+		services.DiffService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithDiffService(s.(diff.DiffClient))
+		},
+		services.NamespacesService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithNamespaceService(s.(namespaces.NamespacesClient))
+		},
+		services.LeasesService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithLeasesService(s.(leases.Manager))
+		},
+		services.IntrospectionService: func(s interface{}) containerd.ServicesOpt {
+			return containerd.WithIntrospectionService(s.(introspectionapi.IntrospectionClient))
+		},
+	} {
+		p := plugins[s]
+		if p == nil {
+			return nil, errors.Errorf("service %q not found", s)
+		}
+		i, err := p.Instance()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get instance of service %q", s)
+		}
+		if i == nil {
+			return nil, errors.Errorf("instance of service %q not found", s)
+		}
+		opts = append(opts, fn(i))
+	}
+	return opts, nil
+}
+
+// Set glog level.
+func setGLogLevel() error {
+	l := logrus.GetLevel()
+	fs := flag.NewFlagSet("klog", flag.PanicOnError)
+	klog.InitFlags(fs)
+	if err := fs.Set("logtostderr", "true"); err != nil {
+		return err
+	}
+	switch l {
+	case logrus.TraceLevel:
+		return fs.Set("v", "5")
+	case logrus.DebugLevel:
+		return fs.Set("v", "4")
+	case logrus.InfoLevel:
+		return fs.Set("v", "2")
+	// glog doesn't support following filters. Defaults to v=0.
+	case logrus.WarnLevel:
+	case logrus.ErrorLevel:
+	case logrus.FatalLevel:
+	case logrus.PanicLevel:
+	}
+	return nil
+}

pkg/ioutil/read_closer.go

@@ -0,0 +1,57 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import "io"
+
+// writeCloseInformer wraps a reader with a close function.
+type wrapReadCloser struct {
+	reader *io.PipeReader
+	writer *io.PipeWriter
+}
+
+// NewWrapReadCloser creates a wrapReadCloser from a reader.
+// NOTE(random-liu): To avoid goroutine leakage, the reader passed in
+// must be eventually closed by the caller.
+func NewWrapReadCloser(r io.Reader) io.ReadCloser {
+	pr, pw := io.Pipe()
+	go func() {
+		_, _ = io.Copy(pw, r)
+		pr.Close()
+		pw.Close()
+	}()
+	return &wrapReadCloser{
+		reader: pr,
+		writer: pw,
+	}
+}
+
+// Read reads up to len(p) bytes into p.
+func (w *wrapReadCloser) Read(p []byte) (int, error) {
+	n, err := w.reader.Read(p)
+	if err == io.ErrClosedPipe {
+		return n, io.EOF
+	}
+	return n, err
+}
+
+// Close closes read closer.
+func (w *wrapReadCloser) Close() error {
+	w.reader.Close()
+	w.writer.Close()
+	return nil
+}

pkg/ioutil/read_closer_test.go

@@ -0,0 +1,47 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWrapReadCloser(t *testing.T) {
+	buf := bytes.NewBufferString("abc")
+
+	rc := NewWrapReadCloser(buf)
+	dst := make([]byte, 1)
+	n, err := rc.Read(dst)
+	assert.Equal(t, 1, n)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("a"), dst)
+
+	n, err = rc.Read(dst)
+	assert.Equal(t, 1, n)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("b"), dst)
+
+	rc.Close()
+	n, err = rc.Read(dst)
+	assert.Equal(t, 0, n)
+	assert.Equal(t, io.EOF, err)
+	assert.Equal(t, []byte("b"), dst)
+}

pkg/ioutil/write_closer.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"io"
+	"sync"
+)
+
+// writeCloseInformer wraps passed in write closer with a close channel.
+// Caller could wait on the close channel for the write closer to be
+// closed.
+type writeCloseInformer struct {
+	close chan struct{}
+	wc    io.WriteCloser
+}
+
+// NewWriteCloseInformer creates the writeCloseInformer from a write closer.
+func NewWriteCloseInformer(wc io.WriteCloser) (io.WriteCloser, <-chan struct{}) {
+	close := make(chan struct{})
+	return &writeCloseInformer{
+		close: close,
+		wc:    wc,
+	}, close
+}
+
+// Write passes through the data into the internal write closer.
+func (w *writeCloseInformer) Write(p []byte) (int, error) {
+	return w.wc.Write(p)
+}
+
+// Close closes the internal write closer and inform the close channel.
+func (w *writeCloseInformer) Close() error {
+	err := w.wc.Close()
+	close(w.close)
+	return err
+}
+
+// nopWriteCloser wraps passed in writer with a nop close function.
+type nopWriteCloser struct {
+	w io.Writer
+}
+
+// NewNopWriteCloser creates the nopWriteCloser from a writer.
+func NewNopWriteCloser(w io.Writer) io.WriteCloser {
+	return &nopWriteCloser{w: w}
+}
+
+// Write passes through the data into the internal writer.
+func (n *nopWriteCloser) Write(p []byte) (int, error) {
+	return n.w.Write(p)
+}
+
+// Close is a nop close function.
+func (n *nopWriteCloser) Close() error {
+	return nil
+}
+
+// serialWriteCloser wraps a write closer and makes sure all writes
+// are done in serial.
+// Parallel write won't intersect with each other. Use case:
+// 1) Pipe: Write content longer than PIPE_BUF.
+//    See http://man7.org/linux/man-pages/man7/pipe.7.html
+// 2) <3.14 Linux Kernel: write is not atomic
+//    See http://man7.org/linux/man-pages/man2/write.2.html
+type serialWriteCloser struct {
+	mu sync.Mutex
+	wc io.WriteCloser
+}
+
+// NewSerialWriteCloser creates a SerialWriteCloser from a write closer.
+func NewSerialWriteCloser(wc io.WriteCloser) io.WriteCloser {
+	return &serialWriteCloser{wc: wc}
+}
+
+// Write writes a group of byte arrays in order atomically.
+func (s *serialWriteCloser) Write(data []byte) (int, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.wc.Write(data)
+}
+
+// Close closes the write closer.
+func (s *serialWriteCloser) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.wc.Close()
+}

pkg/ioutil/write_closer_test.go

@@ -0,0 +1,108 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"io/ioutil"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteCloseInformer(t *testing.T) {
+	original := &writeCloser{}
+	wci, close := NewWriteCloseInformer(original)
+	data := "test"
+
+	n, err := wci.Write([]byte(data))
+	assert.Equal(t, len(data), n)
+	assert.Equal(t, data, original.buf.String())
+	assert.NoError(t, err)
+
+	select {
+	case <-close:
+		assert.Fail(t, "write closer closed")
+	default:
+	}
+
+	wci.Close()
+	assert.True(t, original.closed)
+
+	select {
+	case <-close:
+	default:
+		assert.Fail(t, "write closer not closed")
+	}
+}
+
+func TestSerialWriteCloser(t *testing.T) {
+	const (
+		// Test 10 times to make sure it always pass.
+		testCount = 10
+
+		goroutine = 10
+		dataLen   = 100000
+	)
+	for n := 0; n < testCount; n++ {
+		testData := make([][]byte, goroutine)
+		for i := 0; i < goroutine; i++ {
+			testData[i] = []byte(repeatNumber(i, dataLen) + "\n")
+		}
+
+		f, err := ioutil.TempFile("", "serial-write-closer")
+		require.NoError(t, err)
+		defer os.RemoveAll(f.Name())
+		defer f.Close()
+		wc := NewSerialWriteCloser(f)
+		defer wc.Close()
+
+		// Write data in parallel
+		var wg sync.WaitGroup
+		wg.Add(goroutine)
+		for i := 0; i < goroutine; i++ {
+			go func(id int) {
+				n, err := wc.Write(testData[id])
+				assert.NoError(t, err)
+				assert.Equal(t, dataLen+1, n)
+				wg.Done()
+			}(i)
+		}
+		wg.Wait()
+		wc.Close()
+
+		// Check test result
+		content, err := ioutil.ReadFile(f.Name())
+		require.NoError(t, err)
+		resultData := strings.Split(strings.TrimSpace(string(content)), "\n")
+		require.Len(t, resultData, goroutine)
+		sort.Strings(resultData)
+		for i := 0; i < goroutine; i++ {
+			expected := repeatNumber(i, dataLen)
+			assert.Equal(t, expected, resultData[i])
+		}
+	}
+}
+
+func repeatNumber(num, count int) string {
+	return strings.Repeat(strconv.Itoa(num), count)
+}

pkg/ioutil/writer_group.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"errors"
+	"io"
+	"sync"
+)
+
+// WriterGroup is a group of writers. Writer could be dynamically
+// added and removed.
+type WriterGroup struct {
+	mu      sync.Mutex
+	writers map[string]io.WriteCloser
+	closed  bool
+}
+
+var _ io.Writer = &WriterGroup{}
+
+// NewWriterGroup creates an empty writer group.
+func NewWriterGroup() *WriterGroup {
+	return &WriterGroup{
+		writers: make(map[string]io.WriteCloser),
+	}
+}
+
+// Add adds a writer into the group. The writer will be closed
+// if the writer group is closed.
+func (g *WriterGroup) Add(key string, w io.WriteCloser) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.closed {
+		w.Close()
+		return
+	}
+	g.writers[key] = w
+}
+
+// Get gets a writer from the group, returns nil if the writer
+// doesn't exist.
+func (g *WriterGroup) Get(key string) io.WriteCloser {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.writers[key]
+}
+
+// Remove removes a writer from the group.
+func (g *WriterGroup) Remove(key string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	w, ok := g.writers[key]
+	if !ok {
+		return
+	}
+	w.Close()
+	delete(g.writers, key)
+}
+
+// Write writes data into each writer. If a writer returns error,
+// it will be closed and removed from the writer group. It returns
+// error if writer group is empty.
+func (g *WriterGroup) Write(p []byte) (int, error) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	for k, w := range g.writers {
+		n, err := w.Write(p)
+		if err == nil && len(p) == n {
+			continue
+		}
+		// The writer is closed or in bad state, remove it.
+		w.Close()
+		delete(g.writers, k)
+	}
+	if len(g.writers) == 0 {
+		return 0, errors.New("writer group is empty")
+	}
+	return len(p), nil
+}
+
+// Close closes the writer group. Write will return error after
+// closed.
+func (g *WriterGroup) Close() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	for _, w := range g.writers {
+		w.Close()
+	}
+	g.writers = nil
+	g.closed = true
+}

pkg/ioutil/writer_group_test.go

@@ -0,0 +1,115 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type writeCloser struct {
+	buf    bytes.Buffer
+	closed bool
+}
+
+func (wc *writeCloser) Write(p []byte) (int, error) {
+	return wc.buf.Write(p)
+}
+
+func (wc *writeCloser) Close() error {
+	wc.closed = true
+	return nil
+}
+
+func TestEmptyWriterGroup(t *testing.T) {
+	wg := NewWriterGroup()
+	_, err := wg.Write([]byte("test"))
+	assert.Error(t, err)
+}
+
+func TestClosedWriterGroup(t *testing.T) {
+	wg := NewWriterGroup()
+	wc := &writeCloser{}
+	key, data := "test key", "test data"
+
+	wg.Add(key, wc)
+
+	n, err := wg.Write([]byte(data))
+	assert.Equal(t, len(data), n)
+	assert.Equal(t, data, wc.buf.String())
+	assert.NoError(t, err)
+
+	wg.Close()
+	assert.True(t, wc.closed)
+
+	newWC := &writeCloser{}
+	wg.Add(key, newWC)
+	assert.True(t, newWC.closed)
+
+	_, err = wg.Write([]byte(data))
+	assert.Error(t, err)
+}
+
+func TestAddGetRemoveWriter(t *testing.T) {
+	wg := NewWriterGroup()
+	wc1, wc2 := &writeCloser{}, &writeCloser{}
+	key1, key2 := "test key 1", "test key 2"
+
+	wg.Add(key1, wc1)
+	_, err := wg.Write([]byte("test data 1"))
+	assert.NoError(t, err)
+	assert.Equal(t, "test data 1", wc1.buf.String())
+
+	wg.Add(key2, wc2)
+	_, err = wg.Write([]byte("test data 2"))
+	assert.NoError(t, err)
+	assert.Equal(t, "test data 1test data 2", wc1.buf.String())
+	assert.Equal(t, "test data 2", wc2.buf.String())
+
+	assert.Equal(t, wc1, wg.Get(key1))
+
+	wg.Remove(key1)
+	_, err = wg.Write([]byte("test data 3"))
+	assert.NoError(t, err)
+	assert.Equal(t, "test data 1test data 2", wc1.buf.String())
+	assert.Equal(t, "test data 2test data 3", wc2.buf.String())
+
+	assert.Equal(t, nil, wg.Get(key1))
+
+	wg.Close()
+}
+
+func TestReplaceWriter(t *testing.T) {
+	wg := NewWriterGroup()
+	wc1, wc2 := &writeCloser{}, &writeCloser{}
+	key := "test-key"
+
+	wg.Add(key, wc1)
+	_, err := wg.Write([]byte("test data 1"))
+	assert.NoError(t, err)
+	assert.Equal(t, "test data 1", wc1.buf.String())
+
+	wg.Add(key, wc2)
+	_, err = wg.Write([]byte("test data 2"))
+	assert.NoError(t, err)
+	assert.Equal(t, "test data 1", wc1.buf.String())
+	assert.Equal(t, "test data 2", wc2.buf.String())
+
+	wg.Close()
+}

pkg/netns/netns_linux.go

@@ -0,0 +1,220 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Copyright 2018 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package netns
+
+import (
+	"crypto/rand"
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+	"sync"
+
+	"github.com/containerd/containerd/mount"
+	cnins "github.com/containernetworking/plugins/pkg/ns"
+	"github.com/docker/docker/pkg/symlink"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
+
+const nsRunDir = "/var/run/netns"
+
+// Some of the following functions are migrated from
+// https://github.com/containernetworking/plugins/blob/master/pkg/testutils/netns_linux.go
+
+// newNS creates a new persistent (bind-mounted) network namespace and returns the
+// path to the network namespace.
+func newNS() (nsPath string, err error) {
+	b := make([]byte, 16)
+	if _, err := rand.Reader.Read(b); err != nil {
+		return "", errors.Wrap(err, "failed to generate random netns name")
+	}
+
+	// Create the directory for mounting network namespaces
+	// This needs to be a shared mountpoint in case it is mounted in to
+	// other namespaces (containers)
+	if err := os.MkdirAll(nsRunDir, 0755); err != nil {
+		return "", err
+	}
+
+	// create an empty file at the mount point
+	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	nsPath = path.Join(nsRunDir, nsName)
+	mountPointFd, err := os.Create(nsPath)
+	if err != nil {
+		return "", err
+	}
+	mountPointFd.Close()
+
+	defer func() {
+		// Ensure the mount point is cleaned up on errors
+		if err != nil {
+			os.RemoveAll(nsPath) // nolint: errcheck
+		}
+	}()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// do namespace work in a dedicated goroutine, so that we can safely
+	// Lock/Unlock OSThread without upsetting the lock/unlock state of
+	// the caller of this function
+	go (func() {
+		defer wg.Done()
+		runtime.LockOSThread()
+		// Don't unlock. By not unlocking, golang will kill the OS thread when the
+		// goroutine is done (for go1.10+)
+
+		var origNS cnins.NetNS
+		origNS, err = cnins.GetNS(getCurrentThreadNetNSPath())
+		if err != nil {
+			return
+		}
+		defer origNS.Close()
+
+		// create a new netns on the current thread
+		err = unix.Unshare(unix.CLONE_NEWNET)
+		if err != nil {
+			return
+		}
+
+		// Put this thread back to the orig ns, since it might get reused (pre go1.10)
+		defer origNS.Set() // nolint: errcheck
+
+		// bind mount the netns from the current thread (from /proc) onto the
+		// mount point. This causes the namespace to persist, even when there
+		// are no threads in the ns.
+		err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND, "")
+		if err != nil {
+			err = errors.Wrapf(err, "failed to bind mount ns at %s", nsPath)
+		}
+	})()
+	wg.Wait()
+
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create namespace")
+	}
+
+	return nsPath, nil
+}
+
+// unmountNS unmounts the NS held by the netns object. unmountNS is idempotent.
+func unmountNS(path string) error {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return errors.Wrap(err, "failed to stat netns")
+	}
+	path, err := symlink.FollowSymlinkInScope(path, "/")
+	if err != nil {
+		return errors.Wrap(err, "failed to follow symlink")
+	}
+	if err := mount.Unmount(path, unix.MNT_DETACH); err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "failed to umount netns")
+	}
+	if err := os.RemoveAll(path); err != nil {
+		return errors.Wrap(err, "failed to remove netns")
+	}
+	return nil
+}
+
+// getCurrentThreadNetNSPath copied from pkg/ns
+func getCurrentThreadNetNSPath() string {
+	// /proc/self/ns/net returns the namespace of the main thread, not
+	// of whatever thread this goroutine is running on.  Make sure we
+	// use the thread's net namespace since the thread is switching around
+	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
+}
+
+// NetNS holds network namespace.
+type NetNS struct {
+	path string
+}
+
+// NewNetNS creates a network namespace.
+func NewNetNS() (*NetNS, error) {
+	path, err := newNS()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to setup netns")
+	}
+	return &NetNS{path: path}, nil
+}
+
+// LoadNetNS loads existing network namespace.
+func LoadNetNS(path string) *NetNS {
+	return &NetNS{path: path}
+}
+
+// Remove removes network namepace. Remove is idempotent, meaning it might
+// be invoked multiple times and provides consistent result.
+func (n *NetNS) Remove() error {
+	return unmountNS(n.path)
+}
+
+// Closed checks whether the network namespace has been closed.
+func (n *NetNS) Closed() (bool, error) {
+	ns, err := cnins.GetNS(n.path)
+	if err != nil {
+		if _, ok := err.(cnins.NSPathNotExistErr); ok {
+			// The network namespace has already been removed.
+			return true, nil
+		}
+		if _, ok := err.(cnins.NSPathNotNSErr); ok {
+			// The network namespace is not mounted, remove it.
+			if err := os.RemoveAll(n.path); err != nil {
+				return false, errors.Wrap(err, "remove netns")
+			}
+			return true, nil
+		}
+		return false, errors.Wrap(err, "get netns fd")
+	}
+	if err := ns.Close(); err != nil {
+		return false, errors.Wrap(err, "close netns fd")
+	}
+	return false, nil
+}
+
+// GetPath returns network namespace path for sandbox container
+func (n *NetNS) GetPath() string {
+	return n.path
+}
+
+// Do runs a function in the network namespace.
+func (n *NetNS) Do(f func(cnins.NetNS) error) error {
+	ns, err := cnins.GetNS(n.path)
+	if err != nil {
+		return errors.Wrap(err, "get netns fd")
+	}
+	defer ns.Close() // nolint: errcheck
+	return ns.Do(f)
+}

pkg/netns/netns_other.go

@@ -0,0 +1,58 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package netns
+
+import (
+	"github.com/pkg/errors"
+)
+
+var errNotImplementedOnUnix = errors.New("not implemented on unix")
+
+// NetNS holds network namespace.
+type NetNS struct {
+	path string
+}
+
+// NewNetNS creates a network namespace.
+func NewNetNS() (*NetNS, error) {
+	return nil, errNotImplementedOnUnix
+}
+
+// LoadNetNS loads existing network namespace.
+func LoadNetNS(path string) *NetNS {
+	return &NetNS{path: path}
+}
+
+// Remove removes network namepace. Remove is idempotent, meaning it might
+// be invoked multiple times and provides consistent result.
+func (n *NetNS) Remove() error {
+	return errNotImplementedOnUnix
+}
+
+// Closed checks whether the network namespace has been closed.
+func (n *NetNS) Closed() (bool, error) {
+	return false, errNotImplementedOnUnix
+}
+
+// GetPath returns network namespace path for sandbox container
+func (n *NetNS) GetPath() string {
+	return n.path
+}
+
+// NOTE: Do function is not supported.

pkg/netns/netns_windows.go

@@ -0,0 +1,78 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package netns
+
+import "github.com/Microsoft/hcsshim/hcn"
+
+// NetNS holds network namespace for sandbox
+type NetNS struct {
+	path string
+}
+
+// NewNetNS creates a network namespace for the sandbox
+func NewNetNS() (*NetNS, error) {
+	temp := hcn.HostComputeNamespace{}
+	hcnNamespace, err := temp.Create()
+	if err != nil {
+		return nil, err
+	}
+
+	return &NetNS{path: hcnNamespace.Id}, nil
+}
+
+// LoadNetNS loads existing network namespace.
+func LoadNetNS(path string) *NetNS {
+	return &NetNS{path: path}
+}
+
+// Remove removes network namepace if it exists and not closed. Remove is idempotent,
+// meaning it might be invoked multiple times and provides consistent result.
+func (n *NetNS) Remove() error {
+	hcnNamespace, err := hcn.GetNamespaceByID(n.path)
+	if err != nil {
+		if hcn.IsNotFoundError(err) {
+			return nil
+		}
+		return err
+	}
+	err = hcnNamespace.Delete()
+	if err == nil || hcn.IsNotFoundError(err) {
+		return nil
+	}
+	return err
+}
+
+// Closed checks whether the network namespace has been closed.
+func (n *NetNS) Closed() (bool, error) {
+	_, err := hcn.GetNamespaceByID(n.path)
+	if err == nil {
+		return false, nil
+	}
+	if hcn.IsNotFoundError(err) {
+		return true, nil
+	}
+	return false, err
+}
+
+// GetPath returns network namespace path for sandbox container
+func (n *NetNS) GetPath() string {
+	return n.path
+}
+
+// NOTE: Do function is not supported.

pkg/os/mount_linux.go

@@ -0,0 +1,37 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package os
+
+import (
+	"github.com/containerd/containerd/mount"
+	"golang.org/x/sys/unix"
+)
+
+// Mount will call unix.Mount to mount the file.
+func (RealOS) Mount(source string, target string, fstype string, flags uintptr, data string) error {
+	return unix.Mount(source, target, fstype, flags, data)
+}
+
+// Unmount will call Unmount to unmount the file.
+func (RealOS) Unmount(target string) error {
+	return mount.Unmount(target, unix.MNT_DETACH)
+}
+
+// LookupMount gets mount info of a given path.
+func (RealOS) LookupMount(path string) (mount.Info, error) {
+	return mount.Lookup(path)
+}

pkg/os/mount_other.go

@@ -0,0 +1,26 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package os
+
+import "github.com/containerd/containerd/mount"
+
+// LookupMount gets mount info of a given path.
+func (RealOS) LookupMount(path string) (mount.Info, error) {
+	return mount.Lookup(path)
+}

pkg/os/mount_unix.go

@@ -0,0 +1,33 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package os
+
+import (
+	"github.com/containerd/containerd/mount"
+)
+
+// Mount will call unix.Mount to mount the file.
+func (RealOS) Mount(source string, target string, fstype string, flags uintptr, data string) error {
+	return mount.ErrNotImplementOnUnix
+}
+
+// Unmount will call Unmount to unmount the file.
+func (RealOS) Unmount(target string) error {
+	return mount.Unmount(target, 0)
+}

pkg/os/os.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package os
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/symlink"
+)
+
+// OS collects system level operations that need to be mocked out
+// during tests.
+type OS interface {
+	MkdirAll(path string, perm os.FileMode) error
+	RemoveAll(path string) error
+	Stat(name string) (os.FileInfo, error)
+	ResolveSymbolicLink(name string) (string, error)
+	FollowSymlinkInScope(path, scope string) (string, error)
+	CopyFile(src, dest string, perm os.FileMode) error
+	WriteFile(filename string, data []byte, perm os.FileMode) error
+	Hostname() (string, error)
+}
+
+// RealOS is used to dispatch the real system level operations.
+type RealOS struct{}
+
+// MkdirAll will call os.MkdirAll to create a directory.
+func (RealOS) MkdirAll(path string, perm os.FileMode) error {
+	return os.MkdirAll(path, perm)
+}
+
+// RemoveAll will call os.RemoveAll to remove the path and its children.
+func (RealOS) RemoveAll(path string) error {
+	return os.RemoveAll(path)
+}
+
+// Stat will call os.Stat to get the status of the given file.
+func (RealOS) Stat(name string) (os.FileInfo, error) {
+	return os.Stat(name)
+}
+
+// ResolveSymbolicLink will follow any symbolic links
+func (RealOS) ResolveSymbolicLink(path string) (string, error) {
+	info, err := os.Lstat(path)
+	if err != nil {
+		return "", err
+	}
+	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
+		return path, nil
+	}
+	return filepath.EvalSymlinks(path)
+}
+
+// FollowSymlinkInScope will call symlink.FollowSymlinkInScope.
+func (RealOS) FollowSymlinkInScope(path, scope string) (string, error) {
+	return symlink.FollowSymlinkInScope(path, scope)
+}
+
+// CopyFile will copy src file to dest file
+func (RealOS) CopyFile(src, dest string, perm os.FileMode) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	return err
+}
+
+// WriteFile will call ioutil.WriteFile to write data into a file.
+func (RealOS) WriteFile(filename string, data []byte, perm os.FileMode) error {
+	return ioutil.WriteFile(filename, data, perm)
+}
+
+// Hostname will call os.Hostname to get the hostname of the host.
+func (RealOS) Hostname() (string, error) {
+	return os.Hostname()
+}

pkg/os/os_unix.go

@@ -0,0 +1,31 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package os
+
+import (
+	"github.com/containerd/containerd/mount"
+)
+
+// UNIX collects unix system level operations that need to be
+// mocked out during tests.
+type UNIX interface {
+	Mount(source string, target string, fstype string, flags uintptr, data string) error
+	Unmount(target string) error
+	LookupMount(path string) (mount.Info, error)
+}

pkg/os/testing/fake_os.go

@@ -0,0 +1,254 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testing
+
+import (
+	"os"
+	"sync"
+
+	containerdmount "github.com/containerd/containerd/mount"
+
+	osInterface "github.com/containerd/cri/pkg/os"
+)
+
+// CalledDetail is the struct contains called function name and arguments.
+type CalledDetail struct {
+	// Name of the function called.
+	Name string
+	// Arguments of the function called.
+	Arguments []interface{}
+}
+
+// FakeOS mocks out certain OS calls to avoid perturbing the filesystem
+// If a member of the form `*Fn` is set, that function will be called in place
+// of the real call.
+type FakeOS struct {
+	sync.Mutex
+	MkdirAllFn             func(string, os.FileMode) error
+	RemoveAllFn            func(string) error
+	StatFn                 func(string) (os.FileInfo, error)
+	ResolveSymbolicLinkFn  func(string) (string, error)
+	FollowSymlinkInScopeFn func(string, string) (string, error)
+	CopyFileFn             func(string, string, os.FileMode) error
+	WriteFileFn            func(string, []byte, os.FileMode) error
+	MountFn                func(source string, target string, fstype string, flags uintptr, data string) error
+	UnmountFn              func(target string) error
+	LookupMountFn          func(path string) (containerdmount.Info, error)
+	HostnameFn             func() (string, error)
+	calls                  []CalledDetail
+	errors                 map[string]error
+}
+
+var _ osInterface.OS = &FakeOS{}
+
+// getError get error for call
+func (f *FakeOS) getError(op string) error {
+	f.Lock()
+	defer f.Unlock()
+	err, ok := f.errors[op]
+	if ok {
+		delete(f.errors, op)
+		return err
+	}
+	return nil
+}
+
+// InjectError inject error for call
+func (f *FakeOS) InjectError(fn string, err error) {
+	f.Lock()
+	defer f.Unlock()
+	f.errors[fn] = err
+}
+
+// InjectErrors inject errors for calls
+func (f *FakeOS) InjectErrors(errs map[string]error) {
+	f.Lock()
+	defer f.Unlock()
+	for fn, err := range errs {
+		f.errors[fn] = err
+	}
+}
+
+// ClearErrors clear errors for call
+func (f *FakeOS) ClearErrors() {
+	f.Lock()
+	defer f.Unlock()
+	f.errors = make(map[string]error)
+}
+
+func (f *FakeOS) appendCalls(name string, args ...interface{}) {
+	f.Lock()
+	defer f.Unlock()
+	f.calls = append(f.calls, CalledDetail{Name: name, Arguments: args})
+}
+
+// GetCalls get detail of calls.
+func (f *FakeOS) GetCalls() []CalledDetail {
+	f.Lock()
+	defer f.Unlock()
+	return append([]CalledDetail{}, f.calls...)
+}
+
+// NewFakeOS creates a FakeOS.
+func NewFakeOS() *FakeOS {
+	return &FakeOS{
+		errors: make(map[string]error),
+	}
+}
+
+// MkdirAll is a fake call that invokes MkdirAllFn or just returns nil.
+func (f *FakeOS) MkdirAll(path string, perm os.FileMode) error {
+	f.appendCalls("MkdirAll", path, perm)
+	if err := f.getError("MkdirAll"); err != nil {
+		return err
+	}
+
+	if f.MkdirAllFn != nil {
+		return f.MkdirAllFn(path, perm)
+	}
+	return nil
+}
+
+// RemoveAll is a fake call that invokes RemoveAllFn or just returns nil.
+func (f *FakeOS) RemoveAll(path string) error {
+	f.appendCalls("RemoveAll", path)
+	if err := f.getError("RemoveAll"); err != nil {
+		return err
+	}
+
+	if f.RemoveAllFn != nil {
+		return f.RemoveAllFn(path)
+	}
+	return nil
+}
+
+// Stat is a fake call that invokes StatFn or just return nil.
+func (f *FakeOS) Stat(name string) (os.FileInfo, error) {
+	f.appendCalls("Stat", name)
+	if err := f.getError("Stat"); err != nil {
+		return nil, err
+	}
+
+	if f.StatFn != nil {
+		return f.StatFn(name)
+	}
+	return nil, nil
+}
+
+// ResolveSymbolicLink is a fake call that invokes ResolveSymbolicLinkFn or returns its input
+func (f *FakeOS) ResolveSymbolicLink(path string) (string, error) {
+	f.appendCalls("ResolveSymbolicLink", path)
+	if err := f.getError("ResolveSymbolicLink"); err != nil {
+		return "", err
+	}
+
+	if f.ResolveSymbolicLinkFn != nil {
+		return f.ResolveSymbolicLinkFn(path)
+	}
+	return path, nil
+}
+
+// FollowSymlinkInScope is a fake call that invokes FollowSymlinkInScope or returns its input
+func (f *FakeOS) FollowSymlinkInScope(path, scope string) (string, error) {
+	f.appendCalls("FollowSymlinkInScope", path, scope)
+	if err := f.getError("FollowSymlinkInScope"); err != nil {
+		return "", err
+	}
+
+	if f.FollowSymlinkInScopeFn != nil {
+		return f.FollowSymlinkInScopeFn(path, scope)
+	}
+	return path, nil
+}
+
+// CopyFile is a fake call that invokes CopyFileFn or just return nil.
+func (f *FakeOS) CopyFile(src, dest string, perm os.FileMode) error {
+	f.appendCalls("CopyFile", src, dest, perm)
+	if err := f.getError("CopyFile"); err != nil {
+		return err
+	}
+
+	if f.CopyFileFn != nil {
+		return f.CopyFileFn(src, dest, perm)
+	}
+	return nil
+}
+
+// WriteFile is a fake call that invokes WriteFileFn or just return nil.
+func (f *FakeOS) WriteFile(filename string, data []byte, perm os.FileMode) error {
+	f.appendCalls("WriteFile", filename, data, perm)
+	if err := f.getError("WriteFile"); err != nil {
+		return err
+	}
+
+	if f.WriteFileFn != nil {
+		return f.WriteFileFn(filename, data, perm)
+	}
+	return nil
+}
+
+// Mount is a fake call that invokes MountFn or just return nil.
+func (f *FakeOS) Mount(source string, target string, fstype string, flags uintptr, data string) error {
+	f.appendCalls("Mount", source, target, fstype, flags, data)
+	if err := f.getError("Mount"); err != nil {
+		return err
+	}
+
+	if f.MountFn != nil {
+		return f.MountFn(source, target, fstype, flags, data)
+	}
+	return nil
+}
+
+// Unmount is a fake call that invokes UnmountFn or just return nil.
+func (f *FakeOS) Unmount(target string) error {
+	f.appendCalls("Unmount", target)
+	if err := f.getError("Unmount"); err != nil {
+		return err
+	}
+
+	if f.UnmountFn != nil {
+		return f.UnmountFn(target)
+	}
+	return nil
+}
+
+// LookupMount is a fake call that invokes LookupMountFn or just return nil.
+func (f *FakeOS) LookupMount(path string) (containerdmount.Info, error) {
+	f.appendCalls("LookupMount", path)
+	if err := f.getError("LookupMount"); err != nil {
+		return containerdmount.Info{}, err
+	}
+
+	if f.LookupMountFn != nil {
+		return f.LookupMountFn(path)
+	}
+	return containerdmount.Info{}, nil
+}
+
+// Hostname is a fake call that invokes HostnameFn or just return nil.
+func (f *FakeOS) Hostname() (string, error) {
+	f.appendCalls("Hostname")
+	if err := f.getError("Hostname"); err != nil {
+		return "", err
+	}
+
+	if f.HostnameFn != nil {
+		return f.HostnameFn()
+	}
+	return "", nil
+}

pkg/os/testing/fake_os_unix.go

@@ -0,0 +1,23 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testing
+
+import osInterface "github.com/containerd/cri/pkg/os"
+
+var _ osInterface.UNIX = &FakeOS{}

pkg/registrar/registrar.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package registrar
+
+import (
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+// Registrar stores one-to-one name<->key mappings.
+// Names and keys must be unique.
+// Registrar is safe for concurrent access.
+type Registrar struct {
+	lock      sync.Mutex
+	nameToKey map[string]string
+	keyToName map[string]string
+}
+
+// NewRegistrar creates a new Registrar with the empty indexes.
+func NewRegistrar() *Registrar {
+	return &Registrar{
+		nameToKey: make(map[string]string),
+		keyToName: make(map[string]string),
+	}
+}
+
+// Reserve registers a name<->key mapping, name or key must not
+// be empty.
+// Reserve is idempotent.
+// Attempting to reserve a conflict key<->name mapping results
+// in an error.
+// A name<->key reservation is globally unique.
+func (r *Registrar) Reserve(name, key string) error {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if name == "" || key == "" {
+		return errors.Errorf("invalid name %q or key %q", name, key)
+	}
+
+	if k, exists := r.nameToKey[name]; exists {
+		if k != key {
+			return errors.Errorf("name %q is reserved for %q", name, k)
+		}
+		return nil
+	}
+
+	if n, exists := r.keyToName[key]; exists {
+		if n != name {
+			return errors.Errorf("key %q is reserved for %q", key, n)
+		}
+		return nil
+	}
+
+	r.nameToKey[name] = key
+	r.keyToName[key] = name
+	return nil
+}
+
+// ReleaseByName releases the reserved name<->key mapping by name.
+// Once released, the name and the key can be reserved again.
+func (r *Registrar) ReleaseByName(name string) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	key, exists := r.nameToKey[name]
+	if !exists {
+		return
+	}
+
+	delete(r.nameToKey, name)
+	delete(r.keyToName, key)
+}
+
+// ReleaseByKey release the reserved name<->key mapping by key.
+func (r *Registrar) ReleaseByKey(key string) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	name, exists := r.keyToName[key]
+	if !exists {
+		return
+	}
+
+	delete(r.nameToKey, name)
+	delete(r.keyToName, key)
+}

pkg/registrar/registrar_test.go

@@ -0,0 +1,54 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package registrar
+
+import (
+	"testing"
+
+	assertlib "github.com/stretchr/testify/assert"
+)
+
+func TestRegistrar(t *testing.T) {
+	r := NewRegistrar()
+	assert := assertlib.New(t)
+
+	t.Logf("should be able to reserve a name<->key mapping")
+	assert.NoError(r.Reserve("test-name-1", "test-id-1"))
+
+	t.Logf("should be able to reserve a new name<->key mapping")
+	assert.NoError(r.Reserve("test-name-2", "test-id-2"))
+
+	t.Logf("should be able to reserve the same name<->key mapping")
+	assert.NoError(r.Reserve("test-name-1", "test-id-1"))
+
+	t.Logf("should not be able to reserve conflict name<->key mapping")
+	assert.Error(r.Reserve("test-name-1", "test-id-conflict"))
+	assert.Error(r.Reserve("test-name-conflict", "test-id-2"))
+
+	t.Logf("should be able to release name<->key mapping by key")
+	r.ReleaseByKey("test-id-1")
+
+	t.Logf("should be able to release name<->key mapping by name")
+	r.ReleaseByName("test-name-2")
+
+	t.Logf("should be able to reserve new name<->key mapping after release")
+	assert.NoError(r.Reserve("test-name-1", "test-id-new"))
+	assert.NoError(r.Reserve("test-name-new", "test-id-2"))
+
+	t.Logf("should be able to reserve same name/key name<->key")
+	assert.NoError(r.Reserve("same-name-id", "same-name-id"))
+}

pkg/seccomp/fixtures/proc_self_status

@@ -0,0 +1,47 @@
+Name:   cat
+State:  R (running)
+Tgid:   19383
+Ngid:   0
+Pid:    19383
+PPid:   19275
+TracerPid:  0
+Uid:    1000    1000    1000    1000
+Gid:    1000    1000    1000    1000
+FDSize: 256
+Groups: 24 25 27 29 30 44 46 102 104 108 111 1000 1001
+NStgid: 19383
+NSpid:  19383
+NSpgid: 19383
+NSsid:  19275
+VmPeak:     5944 kB
+VmSize:     5944 kB
+VmLck:         0 kB
+VmPin:         0 kB
+VmHWM:       744 kB
+VmRSS:       744 kB
+VmData:      324 kB
+VmStk:       136 kB
+VmExe:        48 kB
+VmLib:      1776 kB
+VmPTE:        32 kB
+VmPMD:        12 kB
+VmSwap:        0 kB
+Threads:    1
+SigQ:   0/30067
+SigPnd: 0000000000000000
+ShdPnd: 0000000000000000
+SigBlk: 0000000000000000
+SigIgn: 0000000000000080
+SigCgt: 0000000000000000
+CapInh: 0000000000000000
+CapPrm: 0000000000000000
+CapEff: 0000000000000000
+CapBnd: 0000003fffffffff
+CapAmb: 0000000000000000
+Seccomp:    0
+Cpus_allowed:   f
+Cpus_allowed_list:  0-3
+Mems_allowed:   00000000,00000001
+Mems_allowed_list:  0
+voluntary_ctxt_switches:    0
+nonvoluntary_ctxt_switches: 1

pkg/seccomp/seccomp_linux.go

@@ -0,0 +1,88 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright The runc Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package seccomp
+
+import (
+	"bufio"
+	"os"
+	"strings"
+
+	"golang.org/x/sys/unix"
+)
+
+// IsEnabled returns if the kernel has been configured to support seccomp.
+// From https://github.com/opencontainers/runc/blob/v1.0.0-rc91/libcontainer/seccomp/seccomp_linux.go#L86-L102
+func IsEnabled() bool {
+	// Try to read from /proc/self/status for kernels > 3.8
+	s, err := parseStatusFile("/proc/self/status")
+	if err != nil {
+		// Check if Seccomp is supported, via CONFIG_SECCOMP.
+		if err := unix.Prctl(unix.PR_GET_SECCOMP, 0, 0, 0, 0); err != unix.EINVAL {
+			// Make sure the kernel has CONFIG_SECCOMP_FILTER.
+			if err := unix.Prctl(unix.PR_SET_SECCOMP, unix.SECCOMP_MODE_FILTER, 0, 0, 0); err != unix.EINVAL {
+				return true
+			}
+		}
+		return false
+	}
+	_, ok := s["Seccomp"]
+	return ok
+}
+
+// parseStatusFile is from https://github.com/opencontainers/runc/blob/v1.0.0-rc91/libcontainer/seccomp/seccomp_linux.go#L243-L268
+func parseStatusFile(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	s := bufio.NewScanner(f)
+	status := make(map[string]string)
+
+	for s.Scan() {
+		text := s.Text()
+		parts := strings.Split(text, ":")
+
+		if len(parts) <= 1 {
+			continue
+		}
+
+		status[parts[0]] = parts[1]
+	}
+	if err := s.Err(); err != nil {
+		return nil, err
+	}
+
+	return status, nil
+}

pkg/seccomp/seccomp_linux_test.go

@@ -0,0 +1,48 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright The runc Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package seccomp
+
+import "testing"
+
+// TestParseStatusFile is from https://github.com/opencontainers/runc/blob/v1.0.0-rc91/libcontainer/seccomp/seccomp_linux_test.go
+func TestParseStatusFile(t *testing.T) {
+	s, err := parseStatusFile("fixtures/proc_self_status")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := s["Seccomp"]; !ok {
+
+		t.Fatal("expected to find 'Seccomp' in the map but did not.")
+	}
+}

pkg/seccomp/seccomp_unsupported.go

@@ -0,0 +1,23 @@
+// +build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package seccomp
+
+func IsEnabled() bool {
+	return false
+}

pkg/server/bandwidth/doc.go

@@ -0,0 +1,34 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bandwidth provides utilities for bandwidth shaping
+package bandwidth

pkg/server/bandwidth/fake_shaper.go

@@ -0,0 +1,72 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"errors"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// FakeShaper provides an implementation of the bandwith.Shaper.
+// Beware this is implementation has no features besides Reset and GetCIDRs.
+type FakeShaper struct {
+	CIDRs      []string
+	ResetCIDRs []string
+}
+
+// Limit is not implemented
+func (f *FakeShaper) Limit(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+// Reset appends a particular CIDR to the set of ResetCIDRs being managed by this shaper
+func (f *FakeShaper) Reset(cidr string) error {
+	f.ResetCIDRs = append(f.ResetCIDRs, cidr)
+	return nil
+}
+
+// ReconcileInterface is not implemented
+func (f *FakeShaper) ReconcileInterface() error {
+	return errors.New("unimplemented")
+}
+
+// ReconcileCIDR is not implemented
+func (f *FakeShaper) ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+// GetCIDRs returns the set of CIDRs that are being managed by this shaper
+func (f *FakeShaper) GetCIDRs() ([]string, error) {
+	return f.CIDRs, nil
+}

pkg/server/bandwidth/interfaces.go

@@ -0,0 +1,56 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import "k8s.io/apimachinery/pkg/api/resource"
+
+// Shaper is designed so that the shaper structs created
+// satisfy the Shaper interface.
+type Shaper interface {
+	// Limit the bandwidth for a particular CIDR on a particular interface
+	//   * ingress and egress are in bits/second
+	//   * cidr is expected to be a valid network CIDR (e.g. '1.2.3.4/32' or '10.20.0.1/16')
+	// 'egress' bandwidth limit applies to all packets on the interface whose source matches 'cidr'
+	// 'ingress' bandwidth limit applies to all packets on the interface whose destination matches 'cidr'
+	// Limits are aggregate limits for the CIDR, not per IP address.  CIDRs must be unique, but can be overlapping, traffic
+	// that matches multiple CIDRs counts against all limits.
+	Limit(cidr string, egress, ingress *resource.Quantity) error
+	// Remove a bandwidth limit for a particular CIDR on a particular network interface
+	Reset(cidr string) error
+	// Reconcile the interface managed by this shaper with the state on the ground.
+	ReconcileInterface() error
+	// Reconcile a CIDR managed by this shaper with the state on the ground
+	ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error
+	// GetCIDRs returns the set of CIDRs that are being managed by this shaper
+	GetCIDRs() ([]string, error)
+}

pkg/server/bandwidth/linux.go

@@ -0,0 +1,361 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/utils/exec"
+
+	"k8s.io/klog/v2"
+)
+
+var (
+	classShowMatcher      = regexp.MustCompile(`class htb (1:\d+)`)
+	classAndHandleMatcher = regexp.MustCompile(`filter parent 1:.*fh (\d+::\d+).*flowid (\d+:\d+)`)
+)
+
+// tcShaper provides an implementation of the Shaper interface on Linux using the 'tc' tool.
+// In general, using this requires that the caller posses the NET_CAP_ADMIN capability, though if you
+// do this within an container, it only requires the NS_CAPABLE capability for manipulations to that
+// container's network namespace.
+// Uses the hierarchical token bucket queuing discipline (htb), this requires Linux 2.4.20 or newer
+// or a custom kernel with that queuing discipline backported.
+type tcShaper struct {
+	e     exec.Interface
+	iface string
+}
+
+// NewTCShaper makes a new tcShaper for the given interface
+func NewTCShaper(iface string) Shaper {
+	shaper := &tcShaper{
+		e:     exec.New(),
+		iface: iface,
+	}
+	return shaper
+}
+
+func (t *tcShaper) execAndLog(cmdStr string, args ...string) error {
+	klog.V(6).Infof("Running: %s %s", cmdStr, strings.Join(args, " "))
+	cmd := t.e.Command(cmdStr, args...)
+	out, err := cmd.CombinedOutput()
+	klog.V(6).Infof("Output from tc: %s", string(out))
+	return err
+}
+
+func (t *tcShaper) nextClassID() (int, error) {
+	data, err := t.e.Command("tc", "class", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return -1, err
+	}
+
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	classes := sets.String{}
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// skip empty lines
+		if len(line) == 0 {
+			continue
+		}
+		// expected tc line:
+		// class htb 1:1 root prio 0 rate 1000Kbit ceil 1000Kbit burst 1600b cburst 1600b
+		matches := classShowMatcher.FindStringSubmatch(line)
+		if len(matches) != 2 {
+			return -1, fmt.Errorf("unexpected output from tc: %s (%v)", scanner.Text(), matches)
+		}
+		classes.Insert(matches[1])
+	}
+
+	// Make sure it doesn't go forever
+	for nextClass := 1; nextClass < 10000; nextClass++ {
+		if !classes.Has(fmt.Sprintf("1:%d", nextClass)) {
+			return nextClass, nil
+		}
+	}
+	// This should really never happen
+	return -1, fmt.Errorf("exhausted class space, please try again")
+}
+
+// Convert a CIDR from text to a hex representation
+// Strips any masked parts of the IP, so 1.2.3.4/16 becomes hex(1.2.0.0)/ffffffff
+func hexCIDR(cidr string) (string, error) {
+	ip, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return "", err
+	}
+	ip = ip.Mask(ipnet.Mask)
+	hexIP := hex.EncodeToString([]byte(ip))
+	hexMask := ipnet.Mask.String()
+	return hexIP + "/" + hexMask, nil
+}
+
+// Convert a CIDR from hex representation to text, opposite of the above.
+func asciiCIDR(cidr string) (string, error) {
+	parts := strings.Split(cidr, "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected CIDR format: %s", cidr)
+	}
+	ipData, err := hex.DecodeString(parts[0])
+	if err != nil {
+		return "", err
+	}
+	ip := net.IP(ipData)
+
+	maskData, err := hex.DecodeString(parts[1])
+	if err != nil {
+		return "", err
+	}
+	mask := net.IPMask(maskData)
+	size, _ := mask.Size()
+
+	return fmt.Sprintf("%s/%d", ip.String(), size), nil
+}
+
+func (t *tcShaper) findCIDRClass(cidr string) (classAndHandleList [][]string, found bool, err error) {
+	data, err := t.e.Command("tc", "filter", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return classAndHandleList, false, err
+	}
+
+	hex, err := hexCIDR(cidr)
+	if err != nil {
+		return classAndHandleList, false, err
+	}
+	spec := fmt.Sprintf("match %s", hex)
+
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	filter := ""
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		if strings.HasPrefix(line, "filter") {
+			filter = line
+			continue
+		}
+		if strings.Contains(line, spec) {
+			// expected tc line:
+			// `filter parent 1: protocol ip pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1` (old version) or
+			// `filter parent 1: protocol ip pref 1 u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 not_in_hw` (new version)
+			matches := classAndHandleMatcher.FindStringSubmatch(filter)
+			if len(matches) != 3 {
+				return classAndHandleList, false, fmt.Errorf("unexpected output from tc: %s %d (%v)", filter, len(matches), matches)
+			}
+			resultTmp := []string{matches[2], matches[1]}
+			classAndHandleList = append(classAndHandleList, resultTmp)
+		}
+	}
+	if len(classAndHandleList) > 0 {
+		return classAndHandleList, true, nil
+	}
+	return classAndHandleList, false, nil
+}
+
+func makeKBitString(rsrc *resource.Quantity) string {
+	return fmt.Sprintf("%dkbit", (rsrc.Value() / 1000))
+}
+
+func (t *tcShaper) makeNewClass(rate string) (int, error) {
+	class, err := t.nextClassID()
+	if err != nil {
+		return -1, err
+	}
+	if err := t.execAndLog("tc", "class", "add",
+		"dev", t.iface,
+		"parent", "1:",
+		"classid", fmt.Sprintf("1:%d", class),
+		"htb", "rate", rate); err != nil {
+		return -1, err
+	}
+	return class, nil
+}
+
+func (t *tcShaper) Limit(cidr string, upload, download *resource.Quantity) (err error) {
+	var downloadClass, uploadClass int
+	if download != nil {
+		if downloadClass, err = t.makeNewClass(makeKBitString(download)); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "filter", "add",
+			"dev", t.iface,
+			"protocol", "ip",
+			"parent", "1:0",
+			"prio", "1", "u32",
+			"match", "ip", "dst", cidr,
+			"flowid", fmt.Sprintf("1:%d", downloadClass)); err != nil {
+			return err
+		}
+	}
+	if upload != nil {
+		if uploadClass, err = t.makeNewClass(makeKBitString(upload)); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "filter", "add",
+			"dev", t.iface,
+			"protocol", "ip",
+			"parent", "1:0",
+			"prio", "1", "u32",
+			"match", "ip", "src", cidr,
+			"flowid", fmt.Sprintf("1:%d", uploadClass)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// tests to see if an interface exists, if it does, return true and the status line for the interface
+// returns false, "", <err> if an error occurs.
+func (t *tcShaper) interfaceExists() (bool, string, error) {
+	data, err := t.e.Command("tc", "qdisc", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return false, "", err
+	}
+	value := strings.TrimSpace(string(data))
+	if len(value) == 0 {
+		return false, "", nil
+	}
+	// Newer versions of tc and/or the kernel return the following instead of nothing:
+	// qdisc noqueue 0: root refcnt 2
+	fields := strings.Fields(value)
+	if len(fields) > 1 && fields[1] == "noqueue" {
+		return false, "", nil
+	}
+	return true, value, nil
+}
+
+func (t *tcShaper) ReconcileCIDR(cidr string, upload, download *resource.Quantity) error {
+	_, found, err := t.findCIDRClass(cidr)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return t.Limit(cidr, upload, download)
+	}
+	// TODO: actually check bandwidth limits here
+	return nil
+}
+
+func (t *tcShaper) ReconcileInterface() error {
+	exists, output, err := t.interfaceExists()
+	if err != nil {
+		return err
+	}
+	if !exists {
+		klog.V(4).Info("Didn't find bandwidth interface, creating")
+		return t.initializeInterface()
+	}
+	fields := strings.Split(output, " ")
+	if len(fields) < 12 || fields[1] != "htb" || fields[2] != "1:" {
+		if err := t.deleteInterface(fields[2]); err != nil {
+			return err
+		}
+		return t.initializeInterface()
+	}
+	return nil
+}
+
+func (t *tcShaper) initializeInterface() error {
+	return t.execAndLog("tc", "qdisc", "add", "dev", t.iface, "root", "handle", "1:", "htb", "default", "30")
+}
+
+func (t *tcShaper) Reset(cidr string) error {
+	classAndHandle, found, err := t.findCIDRClass(cidr)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("Failed to find cidr: %s on interface: %s", cidr, t.iface)
+	}
+	for i := 0; i < len(classAndHandle); i++ {
+		if err := t.execAndLog("tc", "filter", "del",
+			"dev", t.iface,
+			"parent", "1:",
+			"proto", "ip",
+			"prio", "1",
+			"handle", classAndHandle[i][1], "u32"); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "class", "del",
+			"dev", t.iface,
+			"parent", "1:",
+			"classid", classAndHandle[i][0]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tcShaper) deleteInterface(class string) error {
+	return t.execAndLog("tc", "qdisc", "delete", "dev", t.iface, "root", "handle", class)
+}
+
+func (t *tcShaper) GetCIDRs() ([]string, error) {
+	data, err := t.e.Command("tc", "filter", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+
+	result := []string{}
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		if strings.Contains(line, "match") {
+			parts := strings.Split(line, " ")
+			// expected tc line:
+			// match <cidr> at <number>
+			if len(parts) != 4 {
+				return nil, fmt.Errorf("unexpected output: %v", parts)
+			}
+			cidr, err := asciiCIDR(parts[1])
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, cidr)
+		}
+	}
+	return result, nil
+}

pkg/server/bandwidth/unsupported.go

@@ -0,0 +1,69 @@
+// +build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"errors"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+type unsupportedShaper struct {
+}
+
+// NewTCShaper makes a new unsupportedShapper for the given interface
+func NewTCShaper(iface string) Shaper {
+	return &unsupportedShaper{}
+}
+
+func (f *unsupportedShaper) Limit(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) Reset(cidr string) error {
+	return nil
+}
+
+func (f *unsupportedShaper) ReconcileInterface() error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) GetCIDRs() ([]string, error) {
+	return []string{}, nil
+}

pkg/server/bandwidth/utils.go

@@ -0,0 +1,82 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var minRsrc = resource.MustParse("1k")
+var maxRsrc = resource.MustParse("1P")
+
+func validateBandwidthIsReasonable(rsrc *resource.Quantity) error {
+	if rsrc.Value() < minRsrc.Value() {
+		return fmt.Errorf("resource is unreasonably small (< 1kbit)")
+	}
+	if rsrc.Value() > maxRsrc.Value() {
+		return fmt.Errorf("resoruce is unreasonably large (> 1Pbit)")
+	}
+	return nil
+}
+
+// ExtractPodBandwidthResources extracts the ingress and egress from the given pod annotations
+func ExtractPodBandwidthResources(podAnnotations map[string]string) (ingress, egress *resource.Quantity, err error) {
+	if podAnnotations == nil {
+		return nil, nil, nil
+	}
+	str, found := podAnnotations["kubernetes.io/ingress-bandwidth"]
+	if found {
+		ingressValue, err := resource.ParseQuantity(str)
+		if err != nil {
+			return nil, nil, err
+		}
+		ingress = &ingressValue
+		if err := validateBandwidthIsReasonable(ingress); err != nil {
+			return nil, nil, err
+		}
+	}
+	str, found = podAnnotations["kubernetes.io/egress-bandwidth"]
+	if found {
+		egressValue, err := resource.ParseQuantity(str)
+		if err != nil {
+			return nil, nil, err
+		}
+		egress = &egressValue
+		if err := validateBandwidthIsReasonable(egress); err != nil {
+			return nil, nil, err
+		}
+	}
+	return ingress, egress, nil
+}

pkg/server/cni_conf_syncer.go

@@ -0,0 +1,121 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"sync"
+
+	cni "github.com/containerd/go-cni"
+	"github.com/fsnotify/fsnotify"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// cniNetConfSyncer is used to reload cni network conf triggered by fs change
+// events.
+type cniNetConfSyncer struct {
+	// only used for lastSyncStatus
+	sync.RWMutex
+	lastSyncStatus error
+
+	watcher   *fsnotify.Watcher
+	confDir   string
+	netPlugin cni.CNI
+	loadOpts  []cni.CNIOpt
+}
+
+// newCNINetConfSyncer creates cni network conf syncer.
+func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.CNIOpt) (*cniNetConfSyncer, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create fsnotify watcher")
+	}
+
+	if err := os.MkdirAll(confDir, 0700); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni conf dir=%s for watch", confDir)
+	}
+
+	if err := watcher.Add(confDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to watch cni conf dir %s", confDir)
+	}
+
+	syncer := &cniNetConfSyncer{
+		watcher:   watcher,
+		confDir:   confDir,
+		netPlugin: netPlugin,
+		loadOpts:  loadOpts,
+	}
+
+	if err := syncer.netPlugin.Load(syncer.loadOpts...); err != nil {
+		logrus.WithError(err).Error("failed to load cni during init, please check CRI plugin status before setting up network for pods")
+		syncer.updateLastStatus(err)
+	}
+	return syncer, nil
+}
+
+// syncLoop monitors any fs change events from cni conf dir and tries to reload
+// cni configuration.
+func (syncer *cniNetConfSyncer) syncLoop() error {
+	for {
+		select {
+		case event := <-syncer.watcher.Events:
+			// Only reload config when receiving write/rename/remove
+			// events
+			//
+			// TODO(fuweid): Might only reload target cni config
+			// files to prevent no-ops.
+			if event.Op&(fsnotify.Chmod|fsnotify.Create) > 0 {
+				logrus.Debugf("ignore event from cni conf dir: %s", event)
+				continue
+			}
+			logrus.Debugf("receiving change event from cni conf dir: %s", event)
+
+			lerr := syncer.netPlugin.Load(syncer.loadOpts...)
+			if lerr != nil {
+				logrus.WithError(lerr).
+					Errorf("failed to reload cni configuration after receiving fs change event(%s)", event)
+			}
+			syncer.updateLastStatus(lerr)
+
+		case err := <-syncer.watcher.Errors:
+			if err != nil {
+				logrus.WithError(err).Error("failed to continue sync cni conf change")
+				return err
+			}
+		}
+	}
+}
+
+// lastStatus retrieves last sync status.
+func (syncer *cniNetConfSyncer) lastStatus() error {
+	syncer.RLock()
+	defer syncer.RUnlock()
+	return syncer.lastSyncStatus
+}
+
+// updateLastStatus will be called after every single cni load.
+func (syncer *cniNetConfSyncer) updateLastStatus(err error) {
+	syncer.Lock()
+	defer syncer.Unlock()
+	syncer.lastSyncStatus = err
+}
+
+// stop stops watcher in the syncLoop.
+func (syncer *cniNetConfSyncer) stop() error {
+	return syncer.watcher.Close()
+}

pkg/server/container_attach.go

@@ -0,0 +1,84 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"k8s.io/client-go/tools/remotecommand"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	cio "github.com/containerd/cri/pkg/server/io"
+)
+
+// Attach prepares a streaming endpoint to attach to a running container, and returns the address.
+func (c *criService) Attach(ctx context.Context, r *runtime.AttachRequest) (*runtime.AttachResponse, error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container in store")
+	}
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+	return c.streamServer.GetAttach(r)
+}
+
+func (c *criService) attachContainer(ctx context.Context, id string, stdin io.Reader, stdout, stderr io.WriteCloser,
+	tty bool, resize <-chan remotecommand.TerminalSize) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	// Get container from our container store.
+	cntr, err := c.containerStore.Get(id)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find container %q in store", id)
+	}
+	id = cntr.ID
+
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+
+	task, err := cntr.Container.Task(ctx, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to load task")
+	}
+	handleResizing(ctx, resize, func(size remotecommand.TerminalSize) {
+		if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to resize task %q console", id)
+		}
+	})
+
+	opts := cio.AttachOptions{
+		Stdin:     stdin,
+		Stdout:    stdout,
+		Stderr:    stderr,
+		Tty:       tty,
+		StdinOnce: cntr.Config.StdinOnce,
+		CloseStdin: func() error {
+			return task.CloseIO(ctx, containerd.WithStdinCloser)
+		},
+	}
+	// TODO(random-liu): Figure out whether we need to support historical output.
+	cntr.IO.Attach(opts)
+	return nil
+}

pkg/server/container_create.go

@@ -0,0 +1,343 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/typeurl"
+	"github.com/davecgh/go-spew/spew"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	customopts "github.com/containerd/cri/pkg/containerd/opts"
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	cio "github.com/containerd/cri/pkg/server/io"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	"github.com/containerd/cri/pkg/util"
+)
+
+func init() {
+	typeurl.Register(&containerstore.Metadata{},
+		"github.com/containerd/cri/pkg/store/container", "Metadata")
+}
+
+// CreateContainer creates a new container in the given PodSandbox.
+func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) {
+	config := r.GetConfig()
+	log.G(ctx).Debugf("Container config %+v", config)
+	sandboxConfig := r.GetSandboxConfig()
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find sandbox id %q", r.GetPodSandboxId())
+	}
+	sandboxID := sandbox.ID
+	s, err := sandbox.Container.Task(ctx, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox container task")
+	}
+	sandboxPid := s.Pid()
+
+	// Generate unique id and name for the container and reserve the name.
+	// Reserve the container name to avoid concurrent `CreateContainer` request creating
+	// the same container.
+	id := util.GenerateID()
+	metadata := config.GetMetadata()
+	if metadata == nil {
+		return nil, errors.New("container config must include metadata")
+	}
+	containerName := metadata.Name
+	name := makeContainerName(metadata, sandboxConfig.GetMetadata())
+	log.G(ctx).Debugf("Generated id %q for container %q", id, name)
+	if err = c.containerNameIndex.Reserve(name, id); err != nil {
+		return nil, errors.Wrapf(err, "failed to reserve container name %q", name)
+	}
+	defer func() {
+		// Release the name if the function returns with an error.
+		if retErr != nil {
+			c.containerNameIndex.ReleaseByName(name)
+		}
+	}()
+
+	// Create initial internal container metadata.
+	meta := containerstore.Metadata{
+		ID:        id,
+		Name:      name,
+		SandboxID: sandboxID,
+		Config:    config,
+	}
+
+	// Prepare container image snapshot. For container, the image should have
+	// been pulled before creating the container, so do not ensure the image.
+	image, err := c.localResolve(config.GetImage().GetImage())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to resolve image %q", config.GetImage().GetImage())
+	}
+	containerdImage, err := c.toContainerdImage(ctx, image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get image from containerd %q", image.ID)
+	}
+
+	// Run container using the same runtime with sandbox.
+	sandboxInfo, err := sandbox.Container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
+	}
+
+	// Create container root directory.
+	containerRootDir := c.getContainerRootDir(id)
+	if err = c.os.MkdirAll(containerRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create container root directory %q",
+			containerRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the container root directory.
+			if err = c.os.RemoveAll(containerRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove container root directory %q",
+					containerRootDir)
+			}
+		}
+	}()
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err = c.os.MkdirAll(volatileContainerRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create volatile container root directory %q",
+			volatileContainerRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile container root directory.
+			if err = c.os.RemoveAll(volatileContainerRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove volatile container root directory %q",
+					volatileContainerRootDir)
+			}
+		}
+	}()
+
+	var volumeMounts []*runtime.Mount
+	if !c.config.IgnoreImageDefinedVolumes {
+		// Create container image volumes mounts.
+		volumeMounts = c.volumeMounts(containerRootDir, config.GetMounts(), &image.ImageSpec.Config)
+	} else if len(image.ImageSpec.Config.Volumes) != 0 {
+		log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID)
+	}
+
+	// Generate container mounts.
+	mounts := c.containerMounts(sandboxID, config)
+
+	ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox runtime")
+	}
+	log.G(ctx).Debugf("Use OCI runtime %+v for sandbox %q and container %q", ociRuntime, sandboxID, id)
+
+	spec, err := c.containerSpec(id, sandboxID, sandboxPid, sandbox.NetNSPath, containerName, config, sandboxConfig,
+		&image.ImageSpec.Config, append(mounts, volumeMounts...), ociRuntime)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to generate container %q spec", id)
+	}
+
+	meta.ProcessLabel = spec.Process.SelinuxLabel
+
+	// handle any KVM based runtime
+	if err := modifyProcessLabel(ociRuntime.Type, spec); err != nil {
+		return nil, err
+	}
+
+	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+		// If privileged don't set the SELinux label but still record it on the container so
+		// the unused MCS label can be release later
+		spec.Process.SelinuxLabel = ""
+	}
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(spec.Process.SelinuxLabel)
+		}
+	}()
+
+	log.G(ctx).Debugf("Container %q spec: %#+v", id, spew.NewFormatter(spec))
+
+	// Set snapshotter before any other options.
+	opts := []containerd.NewContainerOpts{
+		containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
+		// Prepare container rootfs. This is always writeable even if
+		// the container wants a readonly rootfs since we want to give
+		// the runtime (runc) a chance to modify (e.g. to create mount
+		// points corresponding to spec.Mounts) before making the
+		// rootfs readonly (requested by spec.Root.Readonly).
+		customopts.WithNewSnapshot(id, containerdImage),
+	}
+	if len(volumeMounts) > 0 {
+		mountMap := make(map[string]string)
+		for _, v := range volumeMounts {
+			mountMap[filepath.Clean(v.HostPath)] = v.ContainerPath
+		}
+		opts = append(opts, customopts.WithVolumes(mountMap))
+	}
+	meta.ImageRef = image.ID
+	meta.StopSignal = image.ImageSpec.Config.StopSignal
+
+	// Validate log paths and compose full container log path.
+	if sandboxConfig.GetLogDirectory() != "" && config.GetLogPath() != "" {
+		meta.LogPath = filepath.Join(sandboxConfig.GetLogDirectory(), config.GetLogPath())
+		log.G(ctx).Debugf("Composed container full log path %q using sandbox log dir %q and container log path %q",
+			meta.LogPath, sandboxConfig.GetLogDirectory(), config.GetLogPath())
+	} else {
+		log.G(ctx).Infof("Logging will be disabled due to empty log paths for sandbox (%q) or container (%q)",
+			sandboxConfig.GetLogDirectory(), config.GetLogPath())
+	}
+
+	containerIO, err := cio.NewContainerIO(id,
+		cio.WithNewFIFOs(volatileContainerRootDir, config.GetTty(), config.GetStdin()))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create container io")
+	}
+	defer func() {
+		if retErr != nil {
+			if err := containerIO.Close(); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to close container io %q", id)
+			}
+		}
+	}()
+
+	specOpts, err := c.containerSpecOpts(config, &image.ImageSpec.Config)
+	if err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	containerLabels := buildLabels(config.Labels, containerKindContainer)
+
+	runtimeOptions, err := getRuntimeOptions(sandboxInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
+	opts = append(opts,
+		containerd.WithSpec(spec, specOpts...),
+		containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
+		containerd.WithContainerLabels(containerLabels),
+		containerd.WithContainerExtension(containerMetadataExtension, &meta))
+	var cntr containerd.Container
+	if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd container")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			if err := cntr.Delete(deferCtx, containerd.WithSnapshotCleanup); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd container %q", id)
+			}
+		}
+	}()
+
+	status := containerstore.Status{CreatedAt: time.Now().UnixNano()}
+	container, err := containerstore.NewContainer(meta,
+		containerstore.WithStatus(status, containerRootDir),
+		containerstore.WithContainer(cntr),
+		containerstore.WithContainerIO(containerIO),
+	)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create internal container object for %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup container checkpoint on error.
+			if err := container.Delete(); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to cleanup container checkpoint for %q", id)
+			}
+		}
+	}()
+
+	// Add container into container store.
+	if err := c.containerStore.Add(container); err != nil {
+		return nil, errors.Wrapf(err, "failed to add container %q into store", id)
+	}
+
+	return &runtime.CreateContainerResponse{ContainerId: id}, nil
+}
+
+// volumeMounts sets up image volumes for container. Rely on the removal of container
+// root directory to do cleanup. Note that image volume will be skipped, if there is criMounts
+// specified with the same destination.
+func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime.Mount, config *imagespec.ImageConfig) []*runtime.Mount {
+	if len(config.Volumes) == 0 {
+		return nil
+	}
+	var mounts []*runtime.Mount
+	for dst := range config.Volumes {
+		if isInCRIMounts(dst, criMounts) {
+			// Skip the image volume, if there is CRI defined volume mapping.
+			// TODO(random-liu): This should be handled by Kubelet in the future.
+			// Kubelet should decide what to use for image volume, and also de-duplicate
+			// the image volume and user mounts.
+			continue
+		}
+		volumeID := util.GenerateID()
+		src := filepath.Join(containerRootDir, "volumes", volumeID)
+		// addOCIBindMounts will create these volumes.
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath:  dst,
+			HostPath:       src,
+			SelinuxRelabel: true,
+		})
+	}
+	return mounts
+}
+
+// runtimeSpec returns a default runtime spec used in cri-containerd.
+func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
+	// GenerateSpec needs namespace.
+	ctx := ctrdutil.NamespacedContext()
+	container := &containers.Container{ID: id}
+
+	if baseSpecFile != "" {
+		baseSpec, ok := c.baseOCISpecs[baseSpecFile]
+		if !ok {
+			return nil, errors.Errorf("can't find base OCI spec %q", baseSpecFile)
+		}
+
+		spec := oci.Spec{}
+		if err := util.DeepCopy(&spec, &baseSpec); err != nil {
+			return nil, errors.Wrap(err, "failed to clone OCI spec")
+		}
+
+		// Fix up cgroups path
+		applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...)
+
+		if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil {
+			return nil, errors.Wrap(err, "failed to apply OCI options")
+		}
+
+		return &spec, nil
+	}
+
+	spec, err := oci.GenerateSpec(ctx, nil, container, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate spec")
+	}
+
+	return spec, nil
+}

pkg/server/container_create_linux.go

@@ -0,0 +1,462 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/cgroups"
+	"github.com/containerd/containerd/contrib/apparmor"
+	"github.com/containerd/containerd/contrib/seccomp"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/annotations"
+	"github.com/containerd/cri/pkg/config"
+	customopts "github.com/containerd/cri/pkg/containerd/opts"
+)
+
+const (
+	// profileNamePrefix is the prefix for loading profiles on a localhost. Eg. AppArmor localhost/profileName.
+	profileNamePrefix = "localhost/" // TODO (mikebrow): get localhost/ & runtime/default from CRI kubernetes/kubernetes#51747
+	// runtimeDefault indicates that we should use or create a runtime default profile.
+	runtimeDefault = "runtime/default"
+	// dockerDefault indicates that we should use or create a docker default profile.
+	dockerDefault = "docker/default"
+	// appArmorDefaultProfileName is name to use when creating a default apparmor profile.
+	appArmorDefaultProfileName = "cri-containerd.apparmor.d"
+	// unconfinedProfile is a string indicating one should run a pod/containerd without a security profile
+	unconfinedProfile = "unconfined"
+	// seccompDefaultProfile is the default seccomp profile.
+	seccompDefaultProfile = dockerDefault
+)
+
+// containerMounts sets up necessary container system file mounts
+// including /dev/shm, /etc/hosts and /etc/resolv.conf.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	var mounts []*runtime.Mount
+	securityContext := config.GetLinux().GetSecurityContext()
+	if !isInCRIMounts(etcHostname, config.GetMounts()) {
+		// /etc/hostname is added since 1.1.6, 1.2.4 and 1.3.
+		// For in-place upgrade, the old sandbox doesn't have the hostname file,
+		// do not mount this in that case.
+		// TODO(random-liu): Remove the check and always mount this when
+		// containerd 1.1 and 1.2 are deprecated.
+		hostpath := c.getSandboxHostname(sandboxID)
+		if _, err := c.os.Stat(hostpath); err == nil {
+			mounts = append(mounts, &runtime.Mount{
+				ContainerPath: etcHostname,
+				HostPath:      hostpath,
+				Readonly:      securityContext.GetReadonlyRootfs(),
+			})
+		}
+	}
+
+	if !isInCRIMounts(etcHosts, config.GetMounts()) {
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: etcHosts,
+			HostPath:      c.getSandboxHosts(sandboxID),
+			Readonly:      securityContext.GetReadonlyRootfs(),
+		})
+	}
+
+	// Mount sandbox resolv.config.
+	// TODO: Need to figure out whether we should always mount it as read-only
+	if !isInCRIMounts(resolvConfPath, config.GetMounts()) {
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: resolvConfPath,
+			HostPath:      c.getResolvPath(sandboxID),
+			Readonly:      securityContext.GetReadonlyRootfs(),
+		})
+	}
+
+	if !isInCRIMounts(devShm, config.GetMounts()) {
+		sandboxDevShm := c.getSandboxDevShm(sandboxID)
+		if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
+			sandboxDevShm = devShm
+		}
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: devShm,
+			HostPath:      sandboxDevShm,
+			Readonly:      false,
+		})
+	}
+	return mounts
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (_ *runtimespec.Spec, retErr error) {
+
+	specOpts := []oci.SpecOpts{
+		customopts.WithoutRunMount,
+		customopts.WithoutDefaultSecuritySettings,
+		customopts.WithRelativeRoot(relativeRootfsPath),
+		customopts.WithProcessArgs(config, imageConfig),
+		oci.WithDefaultPathEnv,
+		// this will be set based on the security context below
+		oci.WithNewPrivileges,
+	}
+	if config.GetWorkingDir() != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir()))
+	} else if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if config.GetTty() {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+
+	// Add HOSTNAME env.
+	var (
+		err      error
+		hostname = sandboxConfig.GetHostname()
+	)
+	if hostname == "" {
+		if hostname, err = c.os.Hostname(); err != nil {
+			return nil, err
+		}
+	}
+	specOpts = append(specOpts, oci.WithEnv([]string{hostnameEnv + "=" + hostname}))
+
+	// Apply envs from image config first, so that envs from container config
+	// can override them.
+	env := imageConfig.Env
+	for _, e := range config.GetEnvs() {
+		env = append(env, e.GetKey()+"="+e.GetValue())
+	}
+	specOpts = append(specOpts, oci.WithEnv(env))
+
+	securityContext := config.GetLinux().GetSecurityContext()
+	labelOptions, err := toLabel(securityContext.GetSelinuxOptions())
+	if err != nil {
+		return nil, err
+	}
+	if len(labelOptions) == 0 {
+		// Use pod level SELinux config
+		if sandbox, err := c.sandboxStore.Get(sandboxID); err == nil {
+			labelOptions, err = selinux.DupSecOpt(sandbox.ProcessLabel)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	processLabel, mountLabel, err := label.InitLabels(labelOptions)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
+	}
+	defer func() {
+		if retErr != nil {
+			_ = label.ReleaseLabel(processLabel)
+		}
+	}()
+
+	specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel))
+
+	if !c.config.DisableProcMount {
+		// Apply masked paths if specified.
+		// If the container is privileged, this will be cleared later on.
+		if maskedPaths := securityContext.GetMaskedPaths(); maskedPaths != nil {
+			specOpts = append(specOpts, oci.WithMaskedPaths(maskedPaths))
+		}
+
+		// Apply readonly paths if specified.
+		// If the container is privileged, this will be cleared later on.
+		if readonlyPaths := securityContext.GetReadonlyPaths(); readonlyPaths != nil {
+			specOpts = append(specOpts, oci.WithReadonlyPaths(readonlyPaths))
+		}
+	}
+
+	if securityContext.GetPrivileged() {
+		if !sandboxConfig.GetLinux().GetSecurityContext().GetPrivileged() {
+			return nil, errors.New("no privileged container allowed in sandbox")
+		}
+		specOpts = append(specOpts, oci.WithPrivileged)
+		if !ociRuntime.PrivilegedWithoutHostDevices {
+			specOpts = append(specOpts, oci.WithHostDevices, oci.WithAllDevicesAllowed)
+		} else {
+			// add requested devices by the config as host devices are not automatically added
+			specOpts = append(specOpts, customopts.WithDevices(c.os, config), customopts.WithCapabilities(securityContext))
+		}
+	} else { // not privileged
+		specOpts = append(specOpts, customopts.WithDevices(c.os, config), customopts.WithCapabilities(securityContext))
+	}
+
+	// Clear all ambient capabilities. The implication of non-root + caps
+	// is not clearly defined in Kubernetes.
+	// See https://github.com/kubernetes/kubernetes/issues/56374
+	// Keep docker's behavior for now.
+	specOpts = append(specOpts,
+		customopts.WithoutAmbientCaps,
+		customopts.WithSelinuxLabels(processLabel, mountLabel),
+	)
+
+	// TODO: Figure out whether we should set no new privilege for sandbox container by default
+	if securityContext.GetNoNewPrivs() {
+		specOpts = append(specOpts, oci.WithNoNewPrivileges)
+	}
+	// TODO(random-liu): [P1] Set selinux options (privileged or not).
+	if securityContext.GetReadonlyRootfs() {
+		specOpts = append(specOpts, oci.WithRootFSReadonly())
+	}
+
+	if c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDisabledCgroups)
+	} else {
+		specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources(), c.config.TolerateMissingHugetlbController, c.config.DisableHugetlbController))
+		if sandboxConfig.GetLinux().GetCgroupParent() != "" {
+			cgroupsPath := getCgroupsPath(sandboxConfig.GetLinux().GetCgroupParent(), id)
+			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
+		}
+	}
+
+	supplementalGroups := securityContext.GetSupplementalGroups()
+
+	for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations,
+		ociRuntime.PodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		ociRuntime.ContainerAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj),
+		customopts.WithPodNamespaces(securityContext, sandboxPid),
+		customopts.WithSupplementalGroups(supplementalGroups),
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer),
+		customopts.WithAnnotation(annotations.SandboxID, sandboxID),
+		customopts.WithAnnotation(annotations.ContainerName, containerName),
+	)
+	// cgroupns is used for hiding /sys/fs/cgroup from containers.
+	// For compatibility, cgroupns is not used when running in cgroup v1 mode or in privileged.
+	// https://github.com/containers/libpod/issues/4363
+	// https://github.com/kubernetes/enhancements/blob/0e409b47497e398b369c281074485c8de129694f/keps/sig-node/20191118-cgroups-v2.md#cgroup-namespace
+	if cgroups.Mode() == cgroups.Unified && !securityContext.GetPrivileged() {
+		specOpts = append(specOpts, oci.WithLinuxNamespace(
+			runtimespec.LinuxNamespace{
+				Type: runtimespec.CgroupNamespace,
+			}))
+	}
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
+}
+
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	var specOpts []oci.SpecOpts
+	securityContext := config.GetLinux().GetSecurityContext()
+	// Set container username. This could only be done by containerd, because it needs
+	// access to the container rootfs. Pass user name to containerd, and let it overwrite
+	// the spec for us.
+	userstr, err := generateUserString(
+		securityContext.GetRunAsUsername(),
+		securityContext.GetRunAsUser(),
+		securityContext.GetRunAsGroup())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate user string")
+	}
+	if userstr == "" {
+		// Lastly, since no user override was passed via CRI try to set via OCI
+		// Image
+		userstr = imageConfig.User
+	}
+	if userstr != "" {
+		specOpts = append(specOpts, oci.WithUser(userstr))
+	}
+
+	if securityContext.GetRunAsUsername() != "" {
+		userstr = securityContext.GetRunAsUsername()
+	} else {
+		// Even if RunAsUser is not set, we still call `GetValue` to get uid 0.
+		// Because it is still useful to get additional gids for uid 0.
+		userstr = strconv.FormatInt(securityContext.GetRunAsUser().GetValue(), 10)
+	}
+	specOpts = append(specOpts, customopts.WithAdditionalGIDs(userstr))
+
+	apparmorSpecOpts, err := generateApparmorSpecOpts(
+		securityContext.GetApparmorProfile(),
+		securityContext.GetPrivileged(),
+		c.apparmorEnabled())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate apparmor spec opts")
+	}
+	if apparmorSpecOpts != nil {
+		specOpts = append(specOpts, apparmorSpecOpts)
+	}
+
+	seccompSpecOpts, err := c.generateSeccompSpecOpts(
+		securityContext.GetSeccompProfilePath(),
+		securityContext.GetPrivileged(),
+		c.seccompEnabled())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate seccomp spec opts")
+	}
+	if seccompSpecOpts != nil {
+		specOpts = append(specOpts, seccompSpecOpts)
+	}
+	return specOpts, nil
+}
+
+// generateSeccompSpecOpts generates containerd SpecOpts for seccomp.
+func (c *criService) generateSeccompSpecOpts(seccompProf string, privileged, seccompEnabled bool) (oci.SpecOpts, error) {
+	if privileged {
+		// Do not set seccomp profile when container is privileged
+		return nil, nil
+	}
+	if seccompProf == "" {
+		seccompProf = c.config.UnsetSeccompProfile
+	}
+	// Set seccomp profile
+	if seccompProf == runtimeDefault || seccompProf == dockerDefault {
+		// use correct default profile (Eg. if not configured otherwise, the default is docker/default)
+		seccompProf = seccompDefaultProfile
+	}
+	if !seccompEnabled {
+		if seccompProf != "" && seccompProf != unconfinedProfile {
+			return nil, errors.New("seccomp is not supported")
+		}
+		return nil, nil
+	}
+	switch seccompProf {
+	case "", unconfinedProfile:
+		// Do not set seccomp profile.
+		return nil, nil
+	case dockerDefault:
+		// Note: WithDefaultProfile specOpts must be added after capabilities
+		return seccomp.WithDefaultProfile(), nil
+	default:
+		// Require and Trim default profile name prefix
+		if !strings.HasPrefix(seccompProf, profileNamePrefix) {
+			return nil, errors.Errorf("invalid seccomp profile %q", seccompProf)
+		}
+		return seccomp.WithProfile(strings.TrimPrefix(seccompProf, profileNamePrefix)), nil
+	}
+}
+
+// generateApparmorSpecOpts generates containerd SpecOpts for apparmor.
+func generateApparmorSpecOpts(apparmorProf string, privileged, apparmorEnabled bool) (oci.SpecOpts, error) {
+	if !apparmorEnabled {
+		// Should fail loudly if user try to specify apparmor profile
+		// but we don't support it.
+		if apparmorProf != "" && apparmorProf != unconfinedProfile {
+			return nil, errors.New("apparmor is not supported")
+		}
+		return nil, nil
+	}
+	switch apparmorProf {
+	// Based on kubernetes#51746, default apparmor profile should be applied
+	// for when apparmor is not specified.
+	case runtimeDefault, "":
+		if privileged {
+			// Do not set apparmor profile when container is privileged
+			return nil, nil
+		}
+		// TODO (mikebrow): delete created apparmor default profile
+		return apparmor.WithDefaultProfile(appArmorDefaultProfileName), nil
+	case unconfinedProfile:
+		return nil, nil
+	default:
+		// Require and Trim default profile name prefix
+		if !strings.HasPrefix(apparmorProf, profileNamePrefix) {
+			return nil, errors.Errorf("invalid apparmor profile %q", apparmorProf)
+		}
+		appArmorProfile := strings.TrimPrefix(apparmorProf, profileNamePrefix)
+		if profileExists, err := appArmorProfileExists(appArmorProfile); !profileExists {
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to generate apparmor spec opts")
+			}
+			return nil, errors.Errorf("apparmor profile not found %s", appArmorProfile)
+		}
+		return apparmor.WithProfile(appArmorProfile), nil
+	}
+}
+
+// appArmorProfileExists scans apparmor/profiles for the requested profile
+func appArmorProfileExists(profile string) (bool, error) {
+	if profile == "" {
+		return false, errors.New("nil apparmor profile is not supported")
+	}
+	profiles, err := os.Open("/sys/kernel/security/apparmor/profiles")
+	if err != nil {
+		return false, err
+	}
+	defer profiles.Close()
+
+	rbuff := bufio.NewReader(profiles)
+	for {
+		line, err := rbuff.ReadString('\n')
+		switch err {
+		case nil:
+			if strings.HasPrefix(line, profile+" (") {
+				return true, nil
+			}
+		case io.EOF:
+			return false, nil
+		default:
+			return false, err
+		}
+	}
+}
+
+// generateUserString generates valid user string based on OCI Image Spec
+// v1.0.0.
+//
+// CRI defines that the following combinations are valid:
+//
+// (none) -> ""
+// username -> username
+// username, uid -> username
+// username, uid, gid -> username:gid
+// username, gid -> username:gid
+// uid -> uid
+// uid, gid -> uid:gid
+// gid -> error
+//
+// TODO(random-liu): Add group name support in CRI.
+func generateUserString(username string, uid, gid *runtime.Int64Value) (string, error) {
+	var userstr, groupstr string
+	if uid != nil {
+		userstr = strconv.FormatInt(uid.GetValue(), 10)
+	}
+	if username != "" {
+		userstr = username
+	}
+	if gid != nil {
+		groupstr = strconv.FormatInt(gid.GetValue(), 10)
+	}
+	if userstr == "" {
+		if groupstr != "" {
+			return "", errors.Errorf("user group %q is specified without user", groupstr)
+		}
+		return "", nil
+	}
+	if groupstr != "" {
+		userstr = userstr + ":" + groupstr
+	}
+	return userstr, nil
+}

pkg/server/container_create_other.go

@@ -0,0 +1,44 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/config"
+)
+
+// containerMounts sets up necessary container system file mounts
+// including /dev/shm, /etc/hosts and /etc/resolv.conf.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	return []*runtime.Mount{}
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (_ *runtimespec.Spec, retErr error) {
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec)
+}
+
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return []oci.SpecOpts{}, nil
+}

pkg/server/container_create_other_test.go

@@ -0,0 +1,40 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// checkMount is defined by all tests but not used here
+var _ = checkMount
+
+func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig,
+	*imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) {
+	config := &runtime.ContainerConfig{}
+	sandboxConfig := &runtime.PodSandboxConfig{}
+	imageConfig := &imagespec.ImageConfig{}
+	specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) {
+	}
+	return config, sandboxConfig, imageConfig, specCheck
+}

pkg/server/container_create_test.go

@@ -0,0 +1,407 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/constants"
+	"github.com/containerd/cri/pkg/containerd/opts"
+)
+
+func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string,
+	contains, notcontains []string) {
+	found := false
+	for _, m := range mounts {
+		if m.Source == src && m.Destination == dest {
+			assert.Equal(t, m.Type, typ)
+			for _, c := range contains {
+				assert.Contains(t, m.Options, c)
+			}
+			for _, n := range notcontains {
+				assert.NotContains(t, m.Options, n)
+			}
+			found = true
+			break
+		}
+	}
+	assert.True(t, found, "mount from %q to %q not found", src, dest)
+}
+
+func TestGeneralContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testPid := uint32(1234)
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	ociRuntime := config.Runtime{}
+	c := newTestCRIService()
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+	require.NoError(t, err)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+}
+
+func TestPodAnnotationPassthroughContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+
+	for desc, test := range map[string]struct {
+		podAnnotations []string
+		configChange   func(*runtime.PodSandboxConfig)
+		specCheck      func(*testing.T, *runtimespec.Spec)
+	}{
+		"a passthrough annotation should be passed as an OCI annotation": {
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+			},
+		},
+		"a non-passthrough annotation should not be passed as an OCI annotation": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["d"] = "e"
+			},
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+				_, ok := spec.Annotations["d"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations should support wildcard match": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["t.f"] = "j"
+				c.Annotations["z.g"] = "o"
+				c.Annotations["z"] = "o"
+				c.Annotations["y.ca"] = "b"
+				c.Annotations["y"] = "b"
+			},
+			podAnnotations: []string{"t*", "z.*", "y.c*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				t.Logf("%+v", spec.Annotations)
+				assert.Equal(t, spec.Annotations["t.f"], "j")
+				assert.Equal(t, spec.Annotations["z.g"], "o")
+				assert.Equal(t, spec.Annotations["y.ca"], "b")
+				_, ok := spec.Annotations["y"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["z"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			c := newTestCRIService()
+			containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+			if test.configChange != nil {
+				test.configChange(sandboxConfig)
+			}
+
+			ociRuntime := config.Runtime{
+				PodAnnotations: test.podAnnotations,
+			}
+			spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName,
+				containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+			assert.NoError(t, err)
+			assert.NotNil(t, spec)
+			specCheck(t, testID, testSandboxID, testPid, spec)
+			if test.specCheck != nil {
+				test.specCheck(t, spec)
+			}
+		})
+	}
+}
+
+func TestContainerSpecCommand(t *testing.T) {
+	for desc, test := range map[string]struct {
+		criEntrypoint   []string
+		criArgs         []string
+		imageEntrypoint []string
+		imageArgs       []string
+		expected        []string
+		expectErr       bool
+	}{
+		"should use cri entrypoint if it's specified": {
+			criEntrypoint:   []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"a", "b"},
+		},
+		"should use cri entrypoint if it's specified even if it's empty": {
+			criEntrypoint:   []string{},
+			criArgs:         []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"a", "b"},
+		},
+		"should use cri entrypoint and args if they are specified": {
+			criEntrypoint:   []string{"a", "b"},
+			criArgs:         []string{"c", "d"},
+			imageEntrypoint: []string{"e", "f"},
+			imageArgs:       []string{"g", "h"},
+			expected:        []string{"a", "b", "c", "d"},
+		},
+		"should use image entrypoint if cri entrypoint is not specified": {
+			criArgs:         []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"c", "d", "a", "b"},
+		},
+		"should use image args if both cri entrypoint and args are not specified": {
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"c", "d", "e", "f"},
+		},
+		"should return error if both entrypoint and args are empty": {
+			expectErr: true,
+		},
+	} {
+
+		config, _, imageConfig, _ := getCreateContainerTestData()
+		config.Command = test.criEntrypoint
+		config.Args = test.criArgs
+		imageConfig.Entrypoint = test.imageEntrypoint
+		imageConfig.Cmd = test.imageArgs
+
+		var spec runtimespec.Spec
+		err := opts.WithProcessArgs(config, imageConfig)(context.Background(), nil, nil, &spec)
+		if test.expectErr {
+			assert.Error(t, err)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, spec.Process.Args, desc)
+	}
+}
+
+func TestVolumeMounts(t *testing.T) {
+	testContainerRootDir := "test-container-root"
+	for desc, test := range map[string]struct {
+		criMounts         []*runtime.Mount
+		imageVolumes      map[string]struct{}
+		expectedMountDest []string
+	}{
+		"should setup rw mount for image volumes": {
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1": {},
+				"/test-volume-2": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-1",
+				"/test-volume-2",
+			},
+		},
+		"should skip image volumes if already mounted by CRI": {
+			criMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/test-volume-1",
+					HostPath:      "/test-hostpath-1",
+				},
+			},
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1": {},
+				"/test-volume-2": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-2",
+			},
+		},
+		"should compare and return cleanpath": {
+			criMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/test-volume-1",
+					HostPath:      "/test-hostpath-1",
+				},
+			},
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1/": {},
+				"/test-volume-2/": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-2/",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		config := &imagespec.ImageConfig{
+			Volumes: test.imageVolumes,
+		}
+		c := newTestCRIService()
+		got := c.volumeMounts(testContainerRootDir, test.criMounts, config)
+		assert.Len(t, got, len(test.expectedMountDest))
+		for _, dest := range test.expectedMountDest {
+			found := false
+			for _, m := range got {
+				if m.ContainerPath == dest {
+					found = true
+					assert.Equal(t,
+						filepath.Dir(m.HostPath),
+						filepath.Join(testContainerRootDir, "volumes"))
+					break
+				}
+			}
+			assert.True(t, found)
+		}
+	}
+}
+
+func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+
+	for desc, test := range map[string]struct {
+		podAnnotations       []string
+		containerAnnotations []string
+		podConfigChange      func(*runtime.PodSandboxConfig)
+		configChange         func(*runtime.ContainerConfig)
+		specCheck            func(*testing.T, *runtimespec.Spec)
+	}{
+		"passthrough annotations from pod and container should be passed as an OCI annotation": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{"pod.annotation.1"},
+			containerAnnotations: []string{"container.annotation.1"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, "1", spec.Annotations["container.annotation.1"])
+				_, ok := spec.Annotations["container.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.3"]
+				assert.False(t, ok)
+				assert.Equal(t, "1", spec.Annotations["pod.annotation.1"])
+				_, ok = spec.Annotations["pod.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.3"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations from pod and container should support wildcard": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{"pod.annotation.*"},
+			containerAnnotations: []string{"container.annotation.*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, "1", spec.Annotations["container.annotation.1"])
+				assert.Equal(t, "2", spec.Annotations["container.annotation.2"])
+				assert.Equal(t, "3", spec.Annotations["container.annotation.3"])
+				assert.Equal(t, "1", spec.Annotations["pod.annotation.1"])
+				assert.Equal(t, "2", spec.Annotations["pod.annotation.2"])
+				assert.Equal(t, "3", spec.Annotations["pod.annotation.3"])
+			},
+		},
+		"annotations should not pass through if no passthrough annotations are configured": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{},
+			containerAnnotations: []string{},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				_, ok := spec.Annotations["container.annotation.1"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.3"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.1"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.3"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			c := newTestCRIService()
+			containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+			if test.configChange != nil {
+				test.configChange(containerConfig)
+			}
+			if test.podConfigChange != nil {
+				test.podConfigChange(sandboxConfig)
+			}
+			ociRuntime := config.Runtime{
+				PodAnnotations:       test.podAnnotations,
+				ContainerAnnotations: test.containerAnnotations,
+			}
+			spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName,
+				containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+			assert.NoError(t, err)
+			assert.NotNil(t, spec)
+			specCheck(t, testID, testSandboxID, testPid, spec)
+			if test.specCheck != nil {
+				test.specCheck(t, spec)
+			}
+		})
+	}
+}
+
+func TestBaseRuntimeSpec(t *testing.T) {
+	c := newTestCRIService()
+	c.baseOCISpecs = map[string]*oci.Spec{
+		"/etc/containerd/cri-base.json": {
+			Version:  "1.0.2",
+			Hostname: "old",
+		},
+	}
+
+	out, err := c.runtimeSpec("id1", "/etc/containerd/cri-base.json", oci.WithHostname("new"))
+	assert.NoError(t, err)
+
+	assert.Equal(t, "1.0.2", out.Version)
+	assert.Equal(t, "new", out.Hostname)
+
+	// Make sure original base spec not changed
+	assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"])
+	assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old")
+
+	assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
+}

pkg/server/container_create_windows.go

@@ -0,0 +1,117 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/annotations"
+	"github.com/containerd/cri/pkg/config"
+	customopts "github.com/containerd/cri/pkg/containerd/opts"
+)
+
+// No container mounts for windows.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	return nil
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (*runtimespec.Spec, error) {
+	specOpts := []oci.SpecOpts{
+		customopts.WithProcessArgs(config, imageConfig),
+	}
+	if config.GetWorkingDir() != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir()))
+	} else if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if config.GetTty() {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+
+	// Apply envs from image config first, so that envs from container config
+	// can override them.
+	env := imageConfig.Env
+	for _, e := range config.GetEnvs() {
+		env = append(env, e.GetKey()+"="+e.GetValue())
+	}
+	specOpts = append(specOpts, oci.WithEnv(env))
+
+	specOpts = append(specOpts,
+		// Clear the root location since hcsshim expects it.
+		// NOTE: readonly rootfs doesn't work on windows.
+		customopts.WithoutRoot,
+		customopts.WithWindowsNetworkNamespace(netNSPath),
+		oci.WithHostname(sandboxConfig.GetHostname()),
+	)
+
+	specOpts = append(specOpts, customopts.WithWindowsMounts(c.os, config, extraMounts))
+
+	// Start with the image config user and override below if RunAsUsername is not "".
+	username := imageConfig.User
+
+	windowsConfig := config.GetWindows()
+	if windowsConfig != nil {
+		specOpts = append(specOpts, customopts.WithWindowsResources(windowsConfig.GetResources()))
+		securityCtx := windowsConfig.GetSecurityContext()
+		if securityCtx != nil {
+			runAsUser := securityCtx.GetRunAsUsername()
+			if runAsUser != "" {
+				username = runAsUser
+			}
+			cs := securityCtx.GetCredentialSpec()
+			if cs != "" {
+				specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs))
+			}
+		}
+	}
+
+	// There really isn't a good Windows way to verify that the username is available in the
+	// image as early as here like there is for Linux. Later on in the stack hcsshim
+	// will handle the behavior of erroring out if the user isn't available in the image
+	// when trying to run the init process.
+	specOpts = append(specOpts, oci.WithUser(username))
+
+	for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations,
+		ociRuntime.PodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		ociRuntime.ContainerAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer),
+		customopts.WithAnnotation(annotations.SandboxID, sandboxID),
+		customopts.WithAnnotation(annotations.ContainerName, containerName),
+	)
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
+}
+
+// No extra spec options needed for windows.
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return nil, nil
+}

pkg/server/container_create_windows_test.go

@@ -0,0 +1,189 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/annotations"
+	"github.com/containerd/cri/pkg/config"
+)
+
+func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig,
+	*imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) {
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image: &runtime.ImageSpec{
+			Image: "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		},
+		Command:    []string{"test", "command"},
+		Args:       []string{"test", "args"},
+		WorkingDir: "test-cwd",
+		Envs: []*runtime.KeyValue{
+			{Key: "k1", Value: "v1"},
+			{Key: "k2", Value: "v2"},
+			{Key: "k3", Value: "v3=v3bis"},
+			{Key: "k4", Value: "v4=v4bis=foop"},
+		},
+		Mounts: []*runtime.Mount{
+			// everything default
+			{
+				ContainerPath: "container-path-1",
+				HostPath:      "host-path-1",
+			},
+			// readOnly
+			{
+				ContainerPath: "container-path-2",
+				HostPath:      "host-path-2",
+				Readonly:      true,
+			},
+		},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+		Windows: &runtime.WindowsContainerConfig{
+			Resources: &runtime.WindowsContainerResources{
+				CpuShares:          100,
+				CpuCount:           200,
+				CpuMaximum:         300,
+				MemoryLimitInBytes: 400,
+			},
+			SecurityContext: &runtime.WindowsContainerSecurityContext{
+				RunAsUsername:  "test-user",
+				CredentialSpec: "{\"test\": \"spec\"}",
+			},
+		},
+	}
+	sandboxConfig := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-sandbox-name",
+			Uid:       "test-sandbox-uid",
+			Namespace: "test-sandbox-ns",
+			Attempt:   2,
+		},
+		Hostname:    "test-hostname",
+		Annotations: map[string]string{"c": "d"},
+	}
+	imageConfig := &imagespec.ImageConfig{
+		Env:        []string{"ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "ik4=iv4=iv4bis=boop"},
+		Entrypoint: []string{"/entrypoint"},
+		Cmd:        []string{"cmd"},
+		WorkingDir: "/workspace",
+		User:       "ContainerUser",
+	}
+	specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) {
+		assert.Nil(t, spec.Root)
+		assert.Equal(t, "test-hostname", spec.Hostname)
+		assert.Equal(t, []string{"test", "command", "test", "args"}, spec.Process.Args)
+		assert.Equal(t, "test-cwd", spec.Process.Cwd)
+		assert.Contains(t, spec.Process.Env, "k1=v1", "k2=v2", "k3=v3=v3bis", "ik4=iv4=iv4bis=boop")
+		assert.Contains(t, spec.Process.Env, "ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "k4=v4=v4bis=foop")
+
+		t.Logf("Check bind mount")
+		checkMount(t, spec.Mounts, "host-path-1", "container-path-1", "", []string{"rw"}, nil)
+		checkMount(t, spec.Mounts, "host-path-2", "container-path-2", "", []string{"ro"}, nil)
+
+		t.Logf("Check resource limits")
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, 100)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Count, 200)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300)
+		assert.EqualValues(t, *spec.Windows.Resources.Memory.Limit, 400)
+
+		// Also checks if override of the image configs user is behaving.
+		t.Logf("Check username")
+		assert.Contains(t, spec.Process.User.Username, "test-user")
+
+		t.Logf("Check credential spec")
+		assert.Contains(t, spec.Windows.CredentialSpec, "{\"test\": \"spec\"}")
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], sandboxID)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeContainer)
+	}
+	return config, sandboxConfig, imageConfig, specCheck
+}
+
+func TestContainerWindowsNetworkNamespace(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	assert.NotNil(t, spec.Windows)
+	assert.NotNil(t, spec.Windows.Network)
+	assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace)
+}
+
+func TestMountCleanPath(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	containerConfig.Mounts = append(containerConfig.Mounts, &runtime.Mount{
+		ContainerPath: "c:/test/container-path",
+		HostPath:      "c:/test/host-path",
+	})
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	checkMount(t, spec.Mounts, "c:\\test\\host-path", "c:\\test\\container-path", "", []string{"rw"}, nil)
+}
+
+func TestMountNamedPipe(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	containerConfig.Mounts = append(containerConfig.Mounts, &runtime.Mount{
+		ContainerPath: `\\.\pipe\foo`,
+		HostPath:      `\\.\pipe\foo`,
+	})
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	checkMount(t, spec.Mounts, `\\.\pipe\foo`, `\\.\pipe\foo`, "", []string{"rw"}, nil)
+}

pkg/server/container_exec.go

@@ -0,0 +1,36 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// Exec prepares a streaming endpoint to execute a command in the container, and returns the address.
+func (c *criService) Exec(ctx context.Context, r *runtime.ExecRequest) (*runtime.ExecResponse, error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find container %q in store", r.GetContainerId())
+	}
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+	return c.streamServer.GetExec(r)
+}

pkg/server/container_execsync.go

@@ -0,0 +1,211 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"bytes"
+	"io"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/oci"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"k8s.io/client-go/tools/remotecommand"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	cioutil "github.com/containerd/cri/pkg/ioutil"
+	cio "github.com/containerd/cri/pkg/server/io"
+	"github.com/containerd/cri/pkg/util"
+)
+
+// ExecSync executes a command in the container, and returns the stdout output.
+// If command exits with a non-zero exit code, an error is returned.
+func (c *criService) ExecSync(ctx context.Context, r *runtime.ExecSyncRequest) (*runtime.ExecSyncResponse, error) {
+	var stdout, stderr bytes.Buffer
+	exitCode, err := c.execInContainer(ctx, r.GetContainerId(), execOptions{
+		cmd:     r.GetCmd(),
+		stdout:  cioutil.NewNopWriteCloser(&stdout),
+		stderr:  cioutil.NewNopWriteCloser(&stderr),
+		timeout: time.Duration(r.GetTimeout()) * time.Second,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to exec in container")
+	}
+
+	return &runtime.ExecSyncResponse{
+		Stdout:   stdout.Bytes(),
+		Stderr:   stderr.Bytes(),
+		ExitCode: int32(*exitCode),
+	}, nil
+}
+
+// execOptions specifies how to execute command in container.
+type execOptions struct {
+	cmd     []string
+	stdin   io.Reader
+	stdout  io.WriteCloser
+	stderr  io.WriteCloser
+	tty     bool
+	resize  <-chan remotecommand.TerminalSize
+	timeout time.Duration
+}
+
+func (c *criService) execInternal(ctx context.Context, container containerd.Container, id string, opts execOptions) (*uint32, error) {
+	// Cancel the context before returning to ensure goroutines are stopped.
+	// This is important, because if `Start` returns error, `Wait` will hang
+	// forever unless we cancel the context.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	spec, err := container.Spec(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container spec")
+	}
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load task")
+	}
+	pspec := spec.Process
+
+	pspec.Terminal = opts.tty
+	if opts.tty {
+		if err := oci.WithEnv([]string{"TERM=xterm"})(ctx, nil, nil, spec); err != nil {
+			return nil, errors.Wrap(err, "add TERM env var to spec")
+		}
+	}
+
+	pspec.Args = opts.cmd
+
+	if opts.stdout == nil {
+		opts.stdout = cio.NewDiscardLogger()
+	}
+	if opts.stderr == nil {
+		opts.stderr = cio.NewDiscardLogger()
+	}
+	execID := util.GenerateID()
+	log.G(ctx).Debugf("Generated exec id %q for container %q", execID, id)
+	volatileRootDir := c.getVolatileContainerRootDir(id)
+	var execIO *cio.ExecIO
+	process, err := task.Exec(ctx, execID, pspec,
+		func(id string) (containerdio.IO, error) {
+			var err error
+			execIO, err = cio.NewExecIO(id, volatileRootDir, opts.tty, opts.stdin != nil)
+			return execIO, err
+		},
+	)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create exec %q", execID)
+	}
+	defer func() {
+		deferCtx, deferCancel := ctrdutil.DeferContext()
+		defer deferCancel()
+		if _, err := process.Delete(deferCtx, containerd.WithProcessKill); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to delete exec process %q for container %q", execID, id)
+		}
+	}()
+
+	exitCh, err := process.Wait(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to wait for process %q", execID)
+	}
+	if err := process.Start(ctx); err != nil {
+		return nil, errors.Wrapf(err, "failed to start exec %q", execID)
+	}
+
+	handleResizing(ctx, opts.resize, func(size remotecommand.TerminalSize) {
+		if err := process.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to resize process %q console for container %q", execID, id)
+		}
+	})
+
+	attachDone := execIO.Attach(cio.AttachOptions{
+		Stdin:     opts.stdin,
+		Stdout:    opts.stdout,
+		Stderr:    opts.stderr,
+		Tty:       opts.tty,
+		StdinOnce: true,
+		CloseStdin: func() error {
+			return process.CloseIO(ctx, containerd.WithStdinCloser)
+		},
+	})
+
+	execCtx := ctx
+	if opts.timeout > 0 {
+		var execCtxCancel context.CancelFunc
+		execCtx, execCtxCancel = context.WithTimeout(ctx, opts.timeout)
+		defer execCtxCancel()
+	}
+
+	select {
+	case <-execCtx.Done():
+		// Ignore the not found error because the process may exit itself before killing.
+		if err := process.Kill(ctx, syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to kill exec %q", execID)
+		}
+		// Wait for the process to be killed.
+		exitRes := <-exitCh
+		log.G(ctx).Infof("Timeout received while waiting for exec process kill %q code %d and error %v",
+			execID, exitRes.ExitCode(), exitRes.Error())
+		<-attachDone
+		log.G(ctx).Debugf("Stream pipe for exec process %q done", execID)
+		return nil, errors.Wrapf(execCtx.Err(), "timeout %v exceeded", opts.timeout)
+	case exitRes := <-exitCh:
+		code, _, err := exitRes.Result()
+		log.G(ctx).Infof("Exec process %q exits with exit code %d and error %v", execID, code, err)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed while waiting for exec %q", execID)
+		}
+		<-attachDone
+		log.G(ctx).Debugf("Stream pipe for exec process %q done", execID)
+		return &code, nil
+	}
+}
+
+// execInContainer executes a command inside the container synchronously, and
+// redirects stdio stream properly.
+// This function only returns when the exec process exits, this means that:
+// 1) As long as the exec process is running, the goroutine in the cri plugin
+// will be running and wait for the exit code;
+// 2) `kubectl exec -it` will hang until the exec process exits, even after io
+// is detached. This is different from dockershim, which leaves the exec process
+// running in background after io is detached.
+// https://github.com/kubernetes/kubernetes/blob/v1.15.0/pkg/kubelet/dockershim/exec.go#L127
+// For example, if the `kubectl exec -it` process is killed, IO will be closed. In
+// this case, the CRI plugin will still have a goroutine waiting for the exec process
+// to exit and log the exit code, but dockershim won't.
+func (c *criService) execInContainer(ctx context.Context, id string, opts execOptions) (*uint32, error) {
+	// Get container from our container store.
+	cntr, err := c.containerStore.Get(id)
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find container %q in store", id)
+	}
+	id = cntr.ID
+
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+
+	return c.execInternal(ctx, cntr.Container, id, opts)
+}

pkg/server/container_list.go

@@ -0,0 +1,112 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// ListContainers lists all containers matching the filter.
+func (c *criService) ListContainers(ctx context.Context, r *runtime.ListContainersRequest) (*runtime.ListContainersResponse, error) {
+	// List all containers from store.
+	containersInStore := c.containerStore.List()
+
+	var containers []*runtime.Container
+	for _, container := range containersInStore {
+		containers = append(containers, toCRIContainer(container))
+	}
+
+	containers = c.filterCRIContainers(containers, r.GetFilter())
+	return &runtime.ListContainersResponse{Containers: containers}, nil
+}
+
+// toCRIContainer converts internal container object into CRI container.
+func toCRIContainer(container containerstore.Container) *runtime.Container {
+	status := container.Status.Get()
+	return &runtime.Container{
+		Id:           container.ID,
+		PodSandboxId: container.SandboxID,
+		Metadata:     container.Config.GetMetadata(),
+		Image:        container.Config.GetImage(),
+		ImageRef:     container.ImageRef,
+		State:        status.State(),
+		CreatedAt:    status.CreatedAt,
+		Labels:       container.Config.GetLabels(),
+		Annotations:  container.Config.GetAnnotations(),
+	}
+}
+
+func (c *criService) normalizeContainerFilter(filter *runtime.ContainerFilter) {
+	if cntr, err := c.containerStore.Get(filter.GetId()); err == nil {
+		filter.Id = cntr.ID
+	}
+	if sb, err := c.sandboxStore.Get(filter.GetPodSandboxId()); err == nil {
+		filter.PodSandboxId = sb.ID
+	}
+}
+
+// filterCRIContainers filters CRIContainers.
+func (c *criService) filterCRIContainers(containers []*runtime.Container, filter *runtime.ContainerFilter) []*runtime.Container {
+	if filter == nil {
+		return containers
+	}
+
+	// The containerd cri plugin supports short ids so long as there is only one
+	// match. So we do a lookup against the store here if a pod id has been
+	// included in the filter.
+	sb := filter.GetPodSandboxId()
+	if sb != "" {
+		sandbox, err := c.sandboxStore.Get(sb)
+		if err == nil {
+			sb = sandbox.ID
+		}
+	}
+
+	c.normalizeContainerFilter(filter)
+	filtered := []*runtime.Container{}
+	for _, cntr := range containers {
+		if filter.GetId() != "" && filter.GetId() != cntr.Id {
+			continue
+		}
+		if sb != "" && sb != cntr.PodSandboxId {
+			continue
+		}
+		if filter.GetState() != nil && filter.GetState().GetState() != cntr.State {
+			continue
+		}
+		if filter.GetLabelSelector() != nil {
+			match := true
+			for k, v := range filter.GetLabelSelector() {
+				got, ok := cntr.Labels[k]
+				if !ok || got != v {
+					match = false
+					break
+				}
+			}
+			if !match {
+				continue
+			}
+		}
+		filtered = append(filtered, cntr)
+	}
+
+	return filtered
+}

pkg/server/container_list_test.go

@@ -0,0 +1,345 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
+)
+
+func TestToCRIContainer(t *testing.T) {
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image:       &runtime.ImageSpec{Image: "test-image"},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+	createdAt := time.Now().UnixNano()
+	container, err := containerstore.NewContainer(
+		containerstore.Metadata{
+			ID:        "test-id",
+			Name:      "test-name",
+			SandboxID: "test-sandbox-id",
+			Config:    config,
+			ImageRef:  "test-image-ref",
+		},
+		containerstore.WithFakeStatus(
+			containerstore.Status{
+				Pid:        1234,
+				CreatedAt:  createdAt,
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+				ExitCode:   1,
+				Reason:     "test-reason",
+				Message:    "test-message",
+			},
+		),
+	)
+	assert.NoError(t, err)
+	expect := &runtime.Container{
+		Id:           "test-id",
+		PodSandboxId: "test-sandbox-id",
+		Metadata:     config.GetMetadata(),
+		Image:        config.GetImage(),
+		ImageRef:     "test-image-ref",
+		State:        runtime.ContainerState_CONTAINER_EXITED,
+		CreatedAt:    createdAt,
+		Labels:       config.GetLabels(),
+		Annotations:  config.GetAnnotations(),
+	}
+	c := toCRIContainer(container)
+	assert.Equal(t, expect, c)
+}
+
+func TestFilterContainers(t *testing.T) {
+	c := newTestCRIService()
+
+	testContainers := []*runtime.Container{
+		{
+			Id:           "1",
+			PodSandboxId: "s-1",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-1", Attempt: 1},
+			State:        runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		{
+			Id:           "2",
+			PodSandboxId: "s-2",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2", Attempt: 2},
+			State:        runtime.ContainerState_CONTAINER_EXITED,
+			Labels:       map[string]string{"a": "b"},
+		},
+		{
+			Id:           "3",
+			PodSandboxId: "s-2",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2", Attempt: 3},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			Labels:       map[string]string{"c": "d"},
+		},
+	}
+	for desc, test := range map[string]struct {
+		filter *runtime.ContainerFilter
+		expect []*runtime.Container
+	}{
+		"no filter": {
+			expect: testContainers,
+		},
+		"id filter": {
+			filter: &runtime.ContainerFilter{Id: "2"},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"state filter": {
+			filter: &runtime.ContainerFilter{
+				State: &runtime.ContainerStateValue{
+					State: runtime.ContainerState_CONTAINER_EXITED,
+				},
+			},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"label filter": {
+			filter: &runtime.ContainerFilter{
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"sandbox id filter": {
+			filter: &runtime.ContainerFilter{PodSandboxId: "s-2"},
+			expect: []*runtime.Container{testContainers[1], testContainers[2]},
+		},
+		"mixed filter not matched": {
+			filter: &runtime.ContainerFilter{
+				Id:            "1",
+				PodSandboxId:  "s-2",
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.Container{},
+		},
+		"mixed filter matched": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-2",
+				State: &runtime.ContainerStateValue{
+					State: runtime.ContainerState_CONTAINER_CREATED,
+				},
+				LabelSelector: map[string]string{"c": "d"},
+			},
+			expect: []*runtime.Container{testContainers[2]},
+		},
+	} {
+		filtered := c.filterCRIContainers(testContainers, test.filter)
+		assert.Equal(t, test.expect, filtered, desc)
+	}
+}
+
+// containerForTest is a helper type for test.
+type containerForTest struct {
+	metadata containerstore.Metadata
+	status   containerstore.Status
+}
+
+func (c containerForTest) toContainer() (containerstore.Container, error) {
+	return containerstore.NewContainer(
+		c.metadata,
+		containerstore.WithFakeStatus(c.status),
+	)
+}
+
+func TestListContainers(t *testing.T) {
+	c := newTestCRIService()
+	sandboxesInStore := []sandboxstore.Sandbox{
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:     "s-1abcdef1234",
+				Name:   "sandboxname-1",
+				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "podname-1"}},
+			},
+			sandboxstore.Status{
+				State: sandboxstore.StateReady,
+			},
+		),
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:     "s-2abcdef1234",
+				Name:   "sandboxname-2",
+				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "podname-2"}},
+			},
+			sandboxstore.Status{
+				State: sandboxstore.StateNotReady,
+			},
+		),
+	}
+	createdAt := time.Now().UnixNano()
+	startedAt := time.Now().UnixNano()
+	finishedAt := time.Now().UnixNano()
+	containersInStore := []containerForTest{
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-1container",
+				Name:      "name-1",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-1"}},
+			},
+			status: containerstore.Status{CreatedAt: createdAt},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-2container",
+				Name:      "name-2",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-2"}},
+			},
+			status: containerstore.Status{
+				CreatedAt: createdAt,
+				StartedAt: startedAt,
+			},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-3container",
+				Name:      "name-3",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-3"}},
+			},
+			status: containerstore.Status{
+				CreatedAt:  createdAt,
+				StartedAt:  startedAt,
+				FinishedAt: finishedAt,
+			},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-4container",
+				Name:      "name-4",
+				SandboxID: "s-2abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-4"}},
+			},
+			status: containerstore.Status{
+				CreatedAt: createdAt,
+			},
+		},
+	}
+
+	expectedContainers := []*runtime.Container{
+		{
+			Id:           "c-1container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-1"},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-2container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2"},
+			State:        runtime.ContainerState_CONTAINER_RUNNING,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-3container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-3"},
+			State:        runtime.ContainerState_CONTAINER_EXITED,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-4container",
+			PodSandboxId: "s-2abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-4"},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			CreatedAt:    createdAt,
+		},
+	}
+
+	// Inject test sandbox metadata
+	for _, sb := range sandboxesInStore {
+		assert.NoError(t, c.sandboxStore.Add(sb))
+	}
+
+	// Inject test container metadata
+	for _, cntr := range containersInStore {
+		container, err := cntr.toContainer()
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(container))
+	}
+
+	for testdesc, testdata := range map[string]struct {
+		filter *runtime.ContainerFilter
+		expect []*runtime.Container
+	}{
+		"test without filter": {
+			filter: &runtime.ContainerFilter{},
+			expect: expectedContainers,
+		},
+		"test filter by sandboxid": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-1abcdef1234",
+			},
+			expect: expectedContainers[:3],
+		},
+		"test filter by truncated sandboxid": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-1",
+			},
+			expect: expectedContainers[:3],
+		},
+		"test filter by containerid": {
+			filter: &runtime.ContainerFilter{
+				Id: "c-1container",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by truncated containerid": {
+			filter: &runtime.ContainerFilter{
+				Id: "c-1",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by containerid and sandboxid": {
+			filter: &runtime.ContainerFilter{
+				Id:           "c-1container",
+				PodSandboxId: "s-1abcdef1234",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by truncated containerid and truncated sandboxid": {
+			filter: &runtime.ContainerFilter{
+				Id:           "c-1",
+				PodSandboxId: "s-1",
+			},
+			expect: expectedContainers[:1],
+		},
+	} {
+		t.Logf("TestCase: %s", testdesc)
+		resp, err := c.ListContainers(context.Background(), &runtime.ListContainersRequest{Filter: testdata.filter})
+		assert.NoError(t, err)
+		require.NotNil(t, resp)
+		containers := resp.GetContainers()
+		assert.Len(t, containers, len(testdata.expect))
+		for _, cntr := range testdata.expect {
+			assert.Contains(t, containers, cntr)
+		}
+	}
+}

pkg/server/container_log_reopen.go

@@ -0,0 +1,51 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ReopenContainerLog asks the cri plugin to reopen the stdout/stderr log file for the container.
+// This is often called after the log file has been rotated.
+func (c *criService) ReopenContainerLog(ctx context.Context, r *runtime.ReopenContainerLogRequest) (*runtime.ReopenContainerLogResponse, error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	if container.Status.Get().State() != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.New("container is not running")
+	}
+
+	// Create new container logger and replace the existing ones.
+	stdoutWC, stderrWC, err := c.createContainerLoggers(container.LogPath, container.Config.GetTty())
+	if err != nil {
+		return nil, err
+	}
+	oldStdoutWC, oldStderrWC := container.IO.AddOutput("log", stdoutWC, stderrWC)
+	if oldStdoutWC != nil {
+		oldStdoutWC.Close()
+	}
+	if oldStderrWC != nil {
+		oldStderrWC.Close()
+	}
+	return &runtime.ReopenContainerLogResponse{}, nil
+}

pkg/server/container_remove.go

@@ -0,0 +1,135 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/store"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// RemoveContainer removes the container.
+func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveContainerRequest) (_ *runtime.RemoveContainerResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+		}
+		// Do not return error if container metadata doesn't exist.
+		log.G(ctx).Tracef("RemoveContainer called for container %q that does not exist", r.GetContainerId())
+		return &runtime.RemoveContainerResponse{}, nil
+	}
+	id := container.ID
+
+	// Forcibly stop the containers if they are in running or unknown state
+	state := container.Status.Get().State()
+	if state == runtime.ContainerState_CONTAINER_RUNNING ||
+		state == runtime.ContainerState_CONTAINER_UNKNOWN {
+		logrus.Infof("Forcibly stopping container %q", id)
+		if err := c.stopContainer(ctx, container, 0); err != nil {
+			return nil, errors.Wrapf(err, "failed to forcibly stop container %q", id)
+		}
+
+	}
+
+	// Set removing state to prevent other start/remove operations against this container
+	// while it's being removed.
+	if err := setContainerRemoving(container); err != nil {
+		return nil, errors.Wrapf(err, "failed to set removing state for container %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Reset removing if remove failed.
+			if err := resetContainerRemoving(container); err != nil {
+				log.G(ctx).WithError(err).Errorf("failed to reset removing state for container %q", id)
+			}
+		}
+	}()
+
+	// NOTE(random-liu): Docker set container to "Dead" state when start removing the
+	// container so as to avoid start/restart the container again. However, for current
+	// kubelet implementation, we'll never start a container once we decide to remove it,
+	// so we don't need the "Dead" state for now.
+
+	// Delete containerd container.
+	if err := container.Container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
+		if !errdefs.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to delete containerd container %q", id)
+		}
+		log.G(ctx).Tracef("Remove called for containerd container %q that does not exist", id)
+	}
+
+	// Delete container checkpoint.
+	if err := container.Delete(); err != nil {
+		return nil, errors.Wrapf(err, "failed to delete container checkpoint for %q", id)
+	}
+
+	containerRootDir := c.getContainerRootDir(id)
+	if err := ensureRemoveAll(ctx, containerRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove container root directory %q",
+			containerRootDir)
+	}
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err := ensureRemoveAll(ctx, volatileContainerRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove volatile container root directory %q",
+			volatileContainerRootDir)
+	}
+
+	c.containerStore.Delete(id)
+
+	c.containerNameIndex.ReleaseByKey(id)
+
+	return &runtime.RemoveContainerResponse{}, nil
+}
+
+// setContainerRemoving sets the container into removing state. In removing state, the
+// container will not be started or removed again.
+func setContainerRemoving(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		// Do not remove container if it's still running or unknown.
+		if status.State() == runtime.ContainerState_CONTAINER_RUNNING {
+			return status, errors.New("container is still running, to stop first")
+		}
+		if status.State() == runtime.ContainerState_CONTAINER_UNKNOWN {
+			return status, errors.New("container state is unknown, to stop first")
+		}
+		if status.Starting {
+			return status, errors.New("container is in starting state, can't be removed")
+		}
+		if status.Removing {
+			return status, errors.New("container is already in removing state")
+		}
+		status.Removing = true
+		return status, nil
+	})
+}
+
+// resetContainerRemoving resets the container removing state on remove failure. So
+// that we could remove the container again.
+func resetContainerRemoving(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Removing = false
+		return status, nil
+	})
+}

pkg/server/container_remove_test.go

@@ -0,0 +1,85 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// TestSetContainerRemoving tests setContainerRemoving sets removing
+// state correctly.
+func TestSetContainerRemoving(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		status    containerstore.Status
+		expectErr bool
+	}{
+		"should return error when container is in running state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in starting state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Starting:  true,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in removing state": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+				Removing:   true,
+			},
+			expectErr: true,
+		},
+		"should not return error when container is not running and removing": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			expectErr: false,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: testID},
+			containerstore.WithFakeStatus(test.status),
+		)
+		assert.NoError(t, err)
+		err = setContainerRemoving(container)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Equal(t, test.status, container.Status.Get(), "metadata should not be updated")
+		} else {
+			assert.NoError(t, err)
+			assert.True(t, container.Status.Get().Removing, "removing should be set")
+			assert.NoError(t, resetContainerRemoving(container))
+			assert.False(t, container.Status.Get().Removing, "removing should be reset")
+		}
+	}
+}

pkg/server/container_start.go

@@ -0,0 +1,223 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io"
+	"time"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	cioutil "github.com/containerd/cri/pkg/ioutil"
+	cio "github.com/containerd/cri/pkg/server/io"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
+)
+
+// StartContainer starts the container.
+func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	id := cntr.ID
+	meta := cntr.Metadata
+	container := cntr.Container
+	config := meta.Config
+
+	// Set starting state to prevent other start/remove operations against this container
+	// while it's being started.
+	if err := setContainerStarting(cntr); err != nil {
+		return nil, errors.Wrapf(err, "failed to set starting state for container %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Set container to exited if fail to start.
+			if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+				status.Pid = 0
+				status.FinishedAt = time.Now().UnixNano()
+				status.ExitCode = errorStartExitCode
+				status.Reason = errorStartReason
+				status.Message = retErr.Error()
+				return status, nil
+			}); err != nil {
+				log.G(ctx).WithError(err).Errorf("failed to set start failure state for container %q", id)
+			}
+		}
+		if err := resetContainerStarting(cntr); err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to reset starting state for container %q", id)
+		}
+	}()
+
+	// Get sandbox config from sandbox store.
+	sandbox, err := c.sandboxStore.Get(meta.SandboxID)
+	if err != nil {
+		return nil, errors.Wrapf(err, "sandbox %q not found", meta.SandboxID)
+	}
+	sandboxID := meta.SandboxID
+	if sandbox.Status.Get().State != sandboxstore.StateReady {
+		return nil, errors.Errorf("sandbox container %q is not running", sandboxID)
+	}
+
+	ioCreation := func(id string) (_ containerdio.IO, err error) {
+		stdoutWC, stderrWC, err := c.createContainerLoggers(meta.LogPath, config.GetTty())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create container loggers")
+		}
+		cntr.IO.AddOutput("log", stdoutWC, stderrWC)
+		cntr.IO.Pipe()
+		return cntr.IO, nil
+	}
+
+	ctrInfo, err := container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container info")
+	}
+
+	taskOpts := c.taskOpts(ctrInfo.Runtime.Name)
+	task, err := container.NewTask(ctx, ioCreation, taskOpts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd task")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// It's possible that task is deleted by event monitor.
+			if _, err := task.Delete(deferCtx, WithNRISandboxDelete(sandboxID), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd task %q", id)
+			}
+		}
+	}()
+
+	// wait is a long running background request, no timeout needed.
+	exitCh, err := task.Wait(ctrdutil.NamespacedContext())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to wait for containerd task")
+	}
+	nric, err := nri.New()
+	if err != nil {
+		log.G(ctx).WithError(err).Error("unable to create nri client")
+	}
+	if nric != nil {
+		nriSB := &nri.Sandbox{
+			ID:     sandboxID,
+			Labels: sandbox.Config.Labels,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil {
+			return nil, errors.Wrap(err, "nri invoke")
+		}
+	}
+
+	// Start containerd task.
+	if err := task.Start(ctx); err != nil {
+		return nil, errors.Wrapf(err, "failed to start containerd task %q", id)
+	}
+
+	// Update container start timestamp.
+	if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Pid = task.Pid()
+		status.StartedAt = time.Now().UnixNano()
+		return status, nil
+	}); err != nil {
+		return nil, errors.Wrapf(err, "failed to update container %q state", id)
+	}
+
+	// start the monitor after updating container state, this ensures that
+	// event monitor receives the TaskExit event and update container state
+	// after this.
+	c.eventMonitor.startExitMonitor(context.Background(), id, task.Pid(), exitCh)
+
+	return &runtime.StartContainerResponse{}, nil
+}
+
+// setContainerStarting sets the container into starting state. In starting state, the
+// container will not be removed or started again.
+func setContainerStarting(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		// Return error if container is not in created state.
+		if status.State() != runtime.ContainerState_CONTAINER_CREATED {
+			return status, errors.Errorf("container is in %s state", criContainerStateToString(status.State()))
+		}
+		// Do not start the container when there is a removal in progress.
+		if status.Removing {
+			return status, errors.New("container is in removing state, can't be started")
+		}
+		if status.Starting {
+			return status, errors.New("container is already in starting state")
+		}
+		status.Starting = true
+		return status, nil
+	})
+}
+
+// resetContainerStarting resets the container starting state on start failure. So
+// that we could remove the container later.
+func resetContainerStarting(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Starting = false
+		return status, nil
+	})
+}
+
+// createContainerLoggers creates container loggers and return write closer for stdout and stderr.
+func (c *criService) createContainerLoggers(logPath string, tty bool) (stdout io.WriteCloser, stderr io.WriteCloser, err error) {
+	if logPath != "" {
+		// Only generate container log when log path is specified.
+		f, err := openLogFile(logPath)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "failed to create and open log file")
+		}
+		defer func() {
+			if err != nil {
+				f.Close()
+			}
+		}()
+		var stdoutCh, stderrCh <-chan struct{}
+		wc := cioutil.NewSerialWriteCloser(f)
+		stdout, stdoutCh = cio.NewCRILogger(logPath, wc, cio.Stdout, c.config.MaxContainerLogLineSize)
+		// Only redirect stderr when there is no tty.
+		if !tty {
+			stderr, stderrCh = cio.NewCRILogger(logPath, wc, cio.Stderr, c.config.MaxContainerLogLineSize)
+		}
+		go func() {
+			if stdoutCh != nil {
+				<-stdoutCh
+			}
+			if stderrCh != nil {
+				<-stderrCh
+			}
+			logrus.Debugf("Finish redirecting log file %q, closing it", logPath)
+			f.Close()
+		}()
+	} else {
+		stdout = cio.NewDiscardLogger()
+		stderr = cio.NewDiscardLogger()
+	}
+	return
+}

pkg/server/container_start_test.go

@@ -0,0 +1,98 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// TestSetContainerStarting tests setContainerStarting sets removing
+// state correctly.
+func TestSetContainerStarting(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		status    containerstore.Status
+		expectErr bool
+	}{
+
+		"should not return error when container is in created state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+			},
+			expectErr: false,
+		},
+		"should return error when container is in running state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in exited state": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in unknown state": {
+			status: containerstore.Status{
+				CreatedAt:  0,
+				StartedAt:  0,
+				FinishedAt: 0,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in starting state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Starting:  true,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in removing state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Removing:  true,
+			},
+			expectErr: true,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: testID},
+			containerstore.WithFakeStatus(test.status),
+		)
+		assert.NoError(t, err)
+		err = setContainerStarting(container)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Equal(t, test.status, container.Status.Get(), "metadata should not be updated")
+		} else {
+			assert.NoError(t, err)
+			assert.True(t, container.Status.Get().Starting, "starting should be set")
+			assert.NoError(t, resetContainerStarting(container))
+			assert.False(t, container.Status.Get().Starting, "starting should be reset")
+		}
+	}
+}

pkg/server/container_stats.go

@@ -0,0 +1,47 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	tasks "github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ContainerStats returns stats of the container. If the container does not
+// exist, the call returns an error.
+func (c *criService) ContainerStats(ctx context.Context, in *runtime.ContainerStatsRequest) (*runtime.ContainerStatsResponse, error) {
+	cntr, err := c.containerStore.Get(in.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	request := &tasks.MetricsRequest{Filters: []string{"id==" + cntr.ID}}
+	resp, err := c.client.TaskService().Metrics(ctx, request)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch metrics for task")
+	}
+	if len(resp.Metrics) != 1 {
+		return nil, errors.Errorf("unexpected metrics response: %+v", resp.Metrics)
+	}
+
+	cs, err := c.containerMetrics(cntr.Metadata, resp.Metrics[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode container metrics")
+	}
+	return &runtime.ContainerStatsResponse{Stats: cs}, nil
+}

pkg/server/container_stats_list.go

@@ -0,0 +1,116 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	tasks "github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/containerd/containerd/api/types"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// ListContainerStats returns stats of all running containers.
+func (c *criService) ListContainerStats(
+	ctx context.Context,
+	in *runtime.ListContainerStatsRequest,
+) (*runtime.ListContainerStatsResponse, error) {
+	request, containers, err := c.buildTaskMetricsRequest(in)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build metrics request")
+	}
+	resp, err := c.client.TaskService().Metrics(ctx, &request)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch metrics for tasks")
+	}
+	criStats, err := c.toCRIContainerStats(resp.Metrics, containers)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to convert to cri containerd stats format")
+	}
+	return criStats, nil
+}
+
+func (c *criService) toCRIContainerStats(
+	stats []*types.Metric,
+	containers []containerstore.Container,
+) (*runtime.ListContainerStatsResponse, error) {
+	statsMap := make(map[string]*types.Metric)
+	for _, stat := range stats {
+		statsMap[stat.ID] = stat
+	}
+	containerStats := new(runtime.ListContainerStatsResponse)
+	for _, cntr := range containers {
+		cs, err := c.containerMetrics(cntr.Metadata, statsMap[cntr.ID])
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to decode container metrics for %q", cntr.ID)
+		}
+		containerStats.Stats = append(containerStats.Stats, cs)
+	}
+	return containerStats, nil
+}
+
+func (c *criService) normalizeContainerStatsFilter(filter *runtime.ContainerStatsFilter) {
+	if cntr, err := c.containerStore.Get(filter.GetId()); err == nil {
+		filter.Id = cntr.ID
+	}
+	if sb, err := c.sandboxStore.Get(filter.GetPodSandboxId()); err == nil {
+		filter.PodSandboxId = sb.ID
+	}
+}
+
+// buildTaskMetricsRequest constructs a tasks.MetricsRequest based on
+// the information in the stats request and the containerStore
+func (c *criService) buildTaskMetricsRequest(
+	r *runtime.ListContainerStatsRequest,
+) (tasks.MetricsRequest, []containerstore.Container, error) {
+	var req tasks.MetricsRequest
+	if r.GetFilter() == nil {
+		return req, nil, nil
+	}
+	c.normalizeContainerStatsFilter(r.GetFilter())
+	var containers []containerstore.Container
+	for _, cntr := range c.containerStore.List() {
+		if r.GetFilter().GetId() != "" && cntr.ID != r.GetFilter().GetId() {
+			continue
+		}
+		if r.GetFilter().GetPodSandboxId() != "" && cntr.SandboxID != r.GetFilter().GetPodSandboxId() {
+			continue
+		}
+		if r.GetFilter().GetLabelSelector() != nil &&
+			!matchLabelSelector(r.GetFilter().GetLabelSelector(), cntr.Config.GetLabels()) {
+			continue
+		}
+		containers = append(containers, cntr)
+		req.Filters = append(req.Filters, "id=="+cntr.ID)
+	}
+	return req, containers, nil
+}
+
+func matchLabelSelector(selector, labels map[string]string) bool {
+	for k, v := range selector {
+		if val, ok := labels[k]; ok {
+			if v != val {
+				return false
+			}
+		} else {
+			return false
+		}
+	}
+	return true
+}

pkg/server/container_stats_list_linux.go

@@ -0,0 +1,127 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+
+	"github.com/containerd/containerd/api/types"
+	v1 "github.com/containerd/containerd/metrics/types/v1"
+	v2 "github.com/containerd/containerd/metrics/types/v2"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.snapshotStore.Get(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		s, err := typeurl.UnmarshalAny(stats.Data)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to extract container metrics")
+		}
+		switch metrics := s.(type) {
+		case *v1.Metrics:
+			if metrics.CPU != nil && metrics.CPU.Usage != nil {
+				cs.Cpu = &runtime.CpuUsage{
+					Timestamp:            stats.Timestamp.UnixNano(),
+					UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total},
+				}
+			}
+			if metrics.Memory != nil && metrics.Memory.Usage != nil {
+				cs.Memory = &runtime.MemoryUsage{
+					Timestamp: stats.Timestamp.UnixNano(),
+					WorkingSetBytes: &runtime.UInt64Value{
+						Value: getWorkingSet(metrics.Memory),
+					},
+				}
+			}
+		case *v2.Metrics:
+			if metrics.CPU != nil {
+				cs.Cpu = &runtime.CpuUsage{
+					Timestamp:            stats.Timestamp.UnixNano(),
+					UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.UsageUsec * 1000},
+				}
+			}
+			if metrics.Memory != nil {
+				cs.Memory = &runtime.MemoryUsage{
+					Timestamp: stats.Timestamp.UnixNano(),
+					WorkingSetBytes: &runtime.UInt64Value{
+						Value: getWorkingSetV2(metrics.Memory),
+					},
+				}
+			}
+		default:
+			return &cs, errors.New(fmt.Sprintf("unxpected metrics type: %v", metrics))
+		}
+	}
+
+	return &cs, nil
+}
+
+// getWorkingSet calculates workingset memory from cgroup memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - total_inactive_file
+func getWorkingSet(memory *v1.MemoryStat) uint64 {
+	if memory.Usage == nil {
+		return 0
+	}
+	var workingSet uint64
+	if memory.TotalInactiveFile < memory.Usage.Usage {
+		workingSet = memory.Usage.Usage - memory.TotalInactiveFile
+	}
+	return workingSet
+}
+
+// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - inactive_file
+func getWorkingSetV2(memory *v2.MemoryStat) uint64 {
+	var workingSet uint64
+	if memory.InactiveFile < memory.Usage {
+		workingSet = memory.Usage - memory.InactiveFile
+	}
+	return workingSet
+}

pkg/server/container_stats_list_linux_test.go

@@ -0,0 +1,55 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	v1 "github.com/containerd/cgroups/stats/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetWorkingSet(t *testing.T) {
+	for desc, test := range map[string]struct {
+		memory   *v1.MemoryStat
+		expected uint64
+	}{
+		"nil memory usage": {
+			memory:   &v1.MemoryStat{},
+			expected: 0,
+		},
+		"memory usage higher than inactive_total_file": {
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 1000,
+				Usage:             &v1.MemoryEntry{Usage: 2000},
+			},
+			expected: 1000,
+		},
+		"memory usage lower than inactive_total_file": {
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 2000,
+				Usage:             &v1.MemoryEntry{Usage: 1000},
+			},
+			expected: 0,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			got := getWorkingSet(test.memory)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}

pkg/server/container_stats_list_other.go

@@ -0,0 +1,36 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	return &cs, errors.Wrap(errdefs.ErrNotImplemented, "container metrics")
+}

pkg/server/container_stats_list_windows.go

@@ -0,0 +1,84 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.snapshotStore.Get(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		s, err := typeurl.UnmarshalAny(stats.Data)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to extract container metrics")
+		}
+		wstats := s.(*wstats.Statistics).GetWindows()
+		if wstats == nil {
+			return nil, errors.New("windows stats is empty")
+		}
+		if wstats.Processor != nil {
+			cs.Cpu = &runtime.CpuUsage{
+				Timestamp:            wstats.Timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS},
+			}
+		}
+		if wstats.Memory != nil {
+			cs.Memory = &runtime.MemoryUsage{
+				Timestamp: wstats.Timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes,
+				},
+			}
+		}
+	}
+	return &cs, nil
+}

pkg/server/container_status.go

@@ -0,0 +1,173 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/store"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// ContainerStatus inspects the container and returns the status.
+func (c *criService) ContainerStatus(ctx context.Context, r *runtime.ContainerStatusRequest) (*runtime.ContainerStatusResponse, error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	// TODO(random-liu): Clean up the following logic in CRI.
+	// Current assumption:
+	// * ImageSpec in container config is image ID.
+	// * ImageSpec in container status is image tag.
+	// * ImageRef in container status is repo digest.
+	spec := container.Config.GetImage()
+	imageRef := container.ImageRef
+	image, err := c.imageStore.Get(imageRef)
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, errors.Wrapf(err, "failed to get image %q", imageRef)
+		}
+	} else {
+		repoTags, repoDigests := parseImageReferences(image.References)
+		if len(repoTags) > 0 {
+			// Based on current behavior of dockershim, this field should be
+			// image tag.
+			spec = &runtime.ImageSpec{Image: repoTags[0]}
+		}
+		if len(repoDigests) > 0 {
+			// Based on the CRI definition, this field will be consumed by user.
+			imageRef = repoDigests[0]
+		}
+	}
+	status := toCRIContainerStatus(container, spec, imageRef)
+	if status.GetCreatedAt() == 0 {
+		// CRI doesn't allow CreatedAt == 0.
+		info, err := container.Container.Info(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get CreatedAt in %q state", status.State)
+		}
+		status.CreatedAt = info.CreatedAt.UnixNano()
+	}
+
+	info, err := toCRIContainerInfo(ctx, container, r.GetVerbose())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get verbose container info")
+	}
+
+	return &runtime.ContainerStatusResponse{
+		Status: status,
+		Info:   info,
+	}, nil
+}
+
+// toCRIContainerStatus converts internal container object to CRI container status.
+func toCRIContainerStatus(container containerstore.Container, spec *runtime.ImageSpec, imageRef string) *runtime.ContainerStatus {
+	meta := container.Metadata
+	status := container.Status.Get()
+	reason := status.Reason
+	if status.State() == runtime.ContainerState_CONTAINER_EXITED && reason == "" {
+		if status.ExitCode == 0 {
+			reason = completeExitReason
+		} else {
+			reason = errorExitReason
+		}
+	}
+
+	return &runtime.ContainerStatus{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		State:       status.State(),
+		CreatedAt:   status.CreatedAt,
+		StartedAt:   status.StartedAt,
+		FinishedAt:  status.FinishedAt,
+		ExitCode:    status.ExitCode,
+		Image:       spec,
+		ImageRef:    imageRef,
+		Reason:      reason,
+		Message:     status.Message,
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+		Mounts:      meta.Config.GetMounts(),
+		LogPath:     meta.LogPath,
+	}
+}
+
+// ContainerInfo is extra information for a container.
+type ContainerInfo struct {
+	// TODO(random-liu): Add sandboxID in CRI container status.
+	SandboxID      string                   `json:"sandboxID"`
+	Pid            uint32                   `json:"pid"`
+	Removing       bool                     `json:"removing"`
+	SnapshotKey    string                   `json:"snapshotKey"`
+	Snapshotter    string                   `json:"snapshotter"`
+	RuntimeType    string                   `json:"runtimeType"`
+	RuntimeOptions interface{}              `json:"runtimeOptions"`
+	Config         *runtime.ContainerConfig `json:"config"`
+	RuntimeSpec    *runtimespec.Spec        `json:"runtimeSpec"`
+}
+
+// toCRIContainerInfo converts internal container object information to CRI container status response info map.
+func toCRIContainerInfo(ctx context.Context, container containerstore.Container, verbose bool) (map[string]string, error) {
+	if !verbose {
+		return nil, nil
+	}
+
+	meta := container.Metadata
+	status := container.Status.Get()
+
+	// TODO(random-liu): Change CRI status info to use array instead of map.
+	ci := &ContainerInfo{
+		SandboxID: container.SandboxID,
+		Pid:       status.Pid,
+		Removing:  status.Removing,
+		Config:    meta.Config,
+	}
+
+	var err error
+	ci.RuntimeSpec, err = container.Container.Spec(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container runtime spec")
+	}
+
+	ctrInfo, err := container.Container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container info")
+	}
+	ci.SnapshotKey = ctrInfo.SnapshotKey
+	ci.Snapshotter = ctrInfo.Snapshotter
+
+	runtimeOptions, err := getRuntimeOptions(ctrInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
+	ci.RuntimeType = ctrInfo.Runtime.Name
+	ci.RuntimeOptions = runtimeOptions
+
+	infoBytes, err := json.Marshal(ci)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal info %v", ci)
+	}
+	return map[string]string{
+		"info": string(infoBytes),
+	}, nil
+}

pkg/server/container_status_test.go

@@ -0,0 +1,227 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	imagestore "github.com/containerd/cri/pkg/store/image"
+)
+
+func getContainerStatusTestData() (*containerstore.Metadata, *containerstore.Status,
+	*imagestore.Image, *runtime.ContainerStatus) {
+	imageID := "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+	testID := "test-id"
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image: &runtime.ImageSpec{Image: "test-image"},
+		Mounts: []*runtime.Mount{{
+			ContainerPath: "test-container-path",
+			HostPath:      "test-host-path",
+		}},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+
+	createdAt := time.Now().UnixNano()
+	startedAt := time.Now().UnixNano()
+
+	metadata := &containerstore.Metadata{
+		ID:        testID,
+		Name:      "test-long-name",
+		SandboxID: "test-sandbox-id",
+		Config:    config,
+		ImageRef:  imageID,
+		LogPath:   "test-log-path",
+	}
+	status := &containerstore.Status{
+		Pid:       1234,
+		CreatedAt: createdAt,
+		StartedAt: startedAt,
+	}
+	image := &imagestore.Image{
+		ID: imageID,
+		References: []string{
+			"gcr.io/library/busybox:latest",
+			"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+	}
+	expected := &runtime.ContainerStatus{
+		Id:          testID,
+		Metadata:    config.GetMetadata(),
+		State:       runtime.ContainerState_CONTAINER_RUNNING,
+		CreatedAt:   createdAt,
+		StartedAt:   startedAt,
+		Image:       &runtime.ImageSpec{Image: "gcr.io/library/busybox:latest"},
+		ImageRef:    "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		Reason:      completeExitReason,
+		Labels:      config.GetLabels(),
+		Annotations: config.GetAnnotations(),
+		Mounts:      config.GetMounts(),
+		LogPath:     "test-log-path",
+	}
+
+	return metadata, status, image, expected
+}
+
+func TestToCRIContainerStatus(t *testing.T) {
+	for desc, test := range map[string]struct {
+		finishedAt     int64
+		exitCode       int32
+		reason         string
+		message        string
+		expectedState  runtime.ContainerState
+		expectedReason string
+	}{
+		"container running": {
+			expectedState: runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		"container exited with reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       1,
+			reason:         "test-reason",
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: "test-reason",
+		},
+		"container exited with exit code 0 without reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       0,
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: completeExitReason,
+		},
+		"container exited with non-zero exit code without reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       1,
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: errorExitReason,
+		},
+	} {
+		metadata, status, _, expected := getContainerStatusTestData()
+		// Update status with test case.
+		status.FinishedAt = test.finishedAt
+		status.ExitCode = test.exitCode
+		status.Reason = test.reason
+		status.Message = test.message
+		container, err := containerstore.NewContainer(
+			*metadata,
+			containerstore.WithFakeStatus(*status),
+		)
+		assert.NoError(t, err)
+		// Set expectation based on test case.
+		expected.State = test.expectedState
+		expected.Reason = test.expectedReason
+		expected.FinishedAt = test.finishedAt
+		expected.ExitCode = test.exitCode
+		expected.Message = test.message
+		containerStatus := toCRIContainerStatus(container,
+			expected.Image,
+			expected.ImageRef)
+		assert.Equal(t, expected, containerStatus, desc)
+	}
+}
+
+// TODO(mikebrow): add a fake containerd container.Container.Spec client api so we can test verbose is true option
+func TestToCRIContainerInfo(t *testing.T) {
+	metadata, status, _, _ := getContainerStatusTestData()
+	container, err := containerstore.NewContainer(
+		*metadata,
+		containerstore.WithFakeStatus(*status),
+	)
+	assert.NoError(t, err)
+
+	info, err := toCRIContainerInfo(context.Background(),
+		container,
+		false)
+	assert.NoError(t, err)
+	assert.Nil(t, info)
+}
+
+func TestContainerStatus(t *testing.T) {
+	for desc, test := range map[string]struct {
+		exist         bool
+		imageExist    bool
+		finishedAt    int64
+		reason        string
+		expectedState runtime.ContainerState
+		expectErr     bool
+	}{
+		"container running": {
+			exist:         true,
+			imageExist:    true,
+			expectedState: runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		"container exited": {
+			exist:         true,
+			imageExist:    true,
+			finishedAt:    time.Now().UnixNano(),
+			reason:        "test-reason",
+			expectedState: runtime.ContainerState_CONTAINER_EXITED,
+		},
+		"container not exist": {
+			exist:      false,
+			imageExist: true,
+			expectErr:  true,
+		},
+		"image not exist": {
+			exist:      false,
+			imageExist: false,
+			expectErr:  true,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		metadata, status, image, expected := getContainerStatusTestData()
+		// Update status with test case.
+		status.FinishedAt = test.finishedAt
+		status.Reason = test.reason
+		container, err := containerstore.NewContainer(
+			*metadata,
+			containerstore.WithFakeStatus(*status),
+		)
+		assert.NoError(t, err)
+		if test.exist {
+			assert.NoError(t, c.containerStore.Add(container))
+		}
+		if test.imageExist {
+			c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{*image})
+			assert.NoError(t, err)
+		}
+		resp, err := c.ContainerStatus(context.Background(), &runtime.ContainerStatusRequest{ContainerId: container.ID})
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Nil(t, resp)
+			continue
+		}
+		// Set expectation based on test case.
+		expected.FinishedAt = test.finishedAt
+		expected.Reason = test.reason
+		expected.State = test.expectedState
+		assert.Equal(t, expected, resp.GetStatus())
+	}
+}

pkg/server/container_stop.go

@@ -0,0 +1,186 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd"
+	eventtypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	"github.com/containerd/cri/pkg/store"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// StopContainer stops a running container with a grace period (i.e., timeout).
+func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainerRequest) (*runtime.StopContainerResponse, error) {
+	// Get container config from container store.
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	if err := c.stopContainer(ctx, container, time.Duration(r.GetTimeout())*time.Second); err != nil {
+		return nil, err
+	}
+
+	return &runtime.StopContainerResponse{}, nil
+}
+
+// stopContainer stops a container based on the container metadata.
+func (c *criService) stopContainer(ctx context.Context, container containerstore.Container, timeout time.Duration) error {
+	id := container.ID
+
+	// Return without error if container is not running. This makes sure that
+	// stop only takes real action after the container is started.
+	state := container.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING &&
+		state != runtime.ContainerState_CONTAINER_UNKNOWN {
+		log.G(ctx).Infof("Container to stop %q must be in running or unknown state, current state %q",
+			id, criContainerStateToString(state))
+		return nil
+	}
+
+	task, err := container.Container.Task(ctx, nil)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to get task for container %q", id)
+		}
+		// Don't return for unknown state, some cleanup needs to be done.
+		if state == runtime.ContainerState_CONTAINER_UNKNOWN {
+			return cleanupUnknownContainer(ctx, id, container)
+		}
+		return nil
+	}
+
+	// Handle unknown state.
+	if state == runtime.ContainerState_CONTAINER_UNKNOWN {
+		// Start an exit handler for containers in unknown state.
+		waitCtx, waitCancel := context.WithCancel(ctrdutil.NamespacedContext())
+		defer waitCancel()
+		exitCh, err := task.Wait(waitCtx)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrapf(err, "failed to wait for task for %q", id)
+			}
+			return cleanupUnknownContainer(ctx, id, container)
+		}
+
+		exitCtx, exitCancel := context.WithCancel(context.Background())
+		stopCh := c.eventMonitor.startExitMonitor(exitCtx, id, task.Pid(), exitCh)
+		defer func() {
+			exitCancel()
+			// This ensures that exit monitor is stopped before
+			// `Wait` is cancelled, so no exit event is generated
+			// because of the `Wait` cancellation.
+			<-stopCh
+		}()
+	}
+
+	// We only need to kill the task. The event handler will Delete the
+	// task from containerd after it handles the Exited event.
+	if timeout > 0 {
+		stopSignal := "SIGTERM"
+		if container.StopSignal != "" {
+			stopSignal = container.StopSignal
+		} else {
+			// The image may have been deleted, and the `StopSignal` field is
+			// just introduced to handle that.
+			// However, for containers created before the `StopSignal` field is
+			// introduced, still try to get the stop signal from the image config.
+			// If the image has been deleted, logging an error and using the
+			// default SIGTERM is still better than returning error and leaving
+			// the container unstoppable. (See issue #990)
+			// TODO(random-liu): Remove this logic when containerd 1.2 is deprecated.
+			image, err := c.imageStore.Get(container.ImageRef)
+			if err != nil {
+				if err != store.ErrNotExist {
+					return errors.Wrapf(err, "failed to get image %q", container.ImageRef)
+				}
+				log.G(ctx).Warningf("Image %q not found, stop container with signal %q", container.ImageRef, stopSignal)
+			} else {
+				if image.ImageSpec.Config.StopSignal != "" {
+					stopSignal = image.ImageSpec.Config.StopSignal
+				}
+			}
+		}
+		sig, err := containerd.ParseSignal(stopSignal)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse stop signal %q", stopSignal)
+		}
+		log.G(ctx).Infof("Stop container %q with signal %v", id, sig)
+		if err = task.Kill(ctx, sig); err != nil && !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to stop container %q", id)
+		}
+
+		sigTermCtx, sigTermCtxCancel := context.WithTimeout(ctx, timeout)
+		defer sigTermCtxCancel()
+		err = c.waitContainerStop(sigTermCtx, container)
+		if err == nil {
+			// Container stopped on first signal no need for SIGKILL
+			return nil
+		}
+		// If the parent context was cancelled or exceeded return immediately
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// sigTermCtx was exceeded. Send SIGKILL
+		log.G(ctx).Debugf("Stop container %q with signal %v timed out", id, sig)
+	}
+
+	log.G(ctx).Infof("Kill container %q", id)
+	if err = task.Kill(ctx, syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrapf(err, "failed to kill container %q", id)
+	}
+
+	// Wait for a fixed timeout until container stop is observed by event monitor.
+	err = c.waitContainerStop(ctx, container)
+	if err != nil {
+		return errors.Wrapf(err, "an error occurs during waiting for container %q to be killed", id)
+	}
+	return nil
+}
+
+// waitContainerStop waits for container to be stopped until context is
+// cancelled or the context deadline is exceeded.
+func (c *criService) waitContainerStop(ctx context.Context, container containerstore.Container) error {
+	select {
+	case <-ctx.Done():
+		return errors.Wrapf(ctx.Err(), "wait container %q", container.ID)
+	case <-container.Stopped():
+		return nil
+	}
+}
+
+// cleanupUnknownContainer cleanup stopped container in unknown state.
+func cleanupUnknownContainer(ctx context.Context, id string, cntr containerstore.Container) error {
+	// Reuse handleContainerExit to do the cleanup.
+	return handleContainerExit(ctx, &eventtypes.TaskExit{
+		ContainerID: id,
+		ID:          id,
+		Pid:         0,
+		ExitStatus:  unknownExitCode,
+		ExitedAt:    time.Now(),
+	}, cntr)
+}

pkg/server/container_stop_test.go

@@ -0,0 +1,85 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+func TestWaitContainerStop(t *testing.T) {
+	id := "test-id"
+	for desc, test := range map[string]struct {
+		status    *containerstore.Status
+		cancel    bool
+		timeout   time.Duration
+		expectErr bool
+	}{
+		"should return error if timeout exceeds": {
+			status: &containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			timeout:   200 * time.Millisecond,
+			expectErr: true,
+		},
+		"should return error if context is cancelled": {
+			status: &containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			timeout:   time.Hour,
+			cancel:    true,
+			expectErr: true,
+		},
+		"should not return error if container is stopped before timeout": {
+			status: &containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			timeout:   time.Hour,
+			expectErr: false,
+		},
+	} {
+		c := newTestCRIService()
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: id},
+			containerstore.WithFakeStatus(*test.status),
+		)
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(container))
+		ctx := context.Background()
+		if test.cancel {
+			cancelledCtx, cancel := context.WithCancel(ctx)
+			cancel()
+			ctx = cancelledCtx
+		}
+		if test.timeout > 0 {
+			timeoutCtx, cancel := context.WithTimeout(ctx, test.timeout)
+			defer cancel()
+			ctx = timeoutCtx
+		}
+		err = c.waitContainerStop(ctx, container)
+		assert.Equal(t, test.expectErr, err != nil, desc)
+	}
+}

pkg/server/container_update_resources_linux.go

@@ -0,0 +1,148 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	gocontext "context"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/typeurl"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/containerd/opts"
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	"github.com/containerd/cri/pkg/util"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (retRes *runtime.UpdateContainerResourcesResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	// Update resources in status update transaction, so that:
+	// 1) There won't be race condition with container start.
+	// 2) There won't be concurrent resource update to the same container.
+	if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		return status, c.updateContainerResources(ctx, container, r.GetLinux(), status)
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to update resources")
+	}
+	return &runtime.UpdateContainerResourcesResponse{}, nil
+}
+
+func (c *criService) updateContainerResources(ctx context.Context,
+	cntr containerstore.Container,
+	resources *runtime.LinuxContainerResources,
+	status containerstore.Status) (retErr error) {
+	id := cntr.ID
+	// Do not update the container when there is a removal in progress.
+	if status.Removing {
+		return errors.Errorf("container %q is in removing state", id)
+	}
+
+	// Update container spec. If the container is not started yet, updating
+	// spec makes sure that the resource limits are correct when start;
+	// if the container is already started, updating spec is still required,
+	// the spec will become our source of truth for resource limits.
+	oldSpec, err := cntr.Container.Spec(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to get container spec")
+	}
+	newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources,
+		c.config.TolerateMissingHugetlbController, c.config.DisableHugetlbController)
+	if err != nil {
+		return errors.Wrap(err, "failed to update resource in spec")
+	}
+
+	if err := updateContainerSpec(ctx, cntr.Container, newSpec); err != nil {
+		return err
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// Reset spec on error.
+			if err := updateContainerSpec(deferCtx, cntr.Container, oldSpec); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to update spec %+v for container %q", oldSpec, id)
+			}
+		}
+	}()
+
+	// If container is not running, only update spec is enough, new resource
+	// limit will be applied when container start.
+	if status.State() != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil
+	}
+
+	task, err := cntr.Container.Task(ctx, nil)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			// Task exited already.
+			return nil
+		}
+		return errors.Wrap(err, "failed to get task")
+	}
+	// newSpec.Linux won't be nil
+	if err := task.Update(ctx, containerd.WithResources(newSpec.Linux.Resources)); err != nil {
+		if errdefs.IsNotFound(err) {
+			// Task exited already.
+			return nil
+		}
+		return errors.Wrap(err, "failed to update resources")
+	}
+	return nil
+}
+
+// updateContainerSpec updates container spec.
+func updateContainerSpec(ctx context.Context, cntr containerd.Container, spec *runtimespec.Spec) error {
+	any, err := typeurl.MarshalAny(spec)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal spec %+v", spec)
+	}
+	if err := cntr.Update(ctx, func(ctx gocontext.Context, client *containerd.Client, c *containers.Container) error {
+		c.Spec = any
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "failed to update container spec")
+	}
+	return nil
+}
+
+// updateOCILinuxResource updates container resource limit.
+func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources,
+	tolerateMissingHugetlbController, disableHugetlbController bool) (*runtimespec.Spec, error) {
+	// Copy to make sure old spec is not changed.
+	var cloned runtimespec.Spec
+	if err := util.DeepCopy(&cloned, spec); err != nil {
+		return nil, errors.Wrap(err, "failed to deep copy")
+	}
+	if cloned.Linux == nil {
+		cloned.Linux = &runtimespec.Linux{}
+	}
+	if err := opts.WithResources(new, tolerateMissingHugetlbController, disableHugetlbController)(ctx, nil, nil, &cloned); err != nil {
+		return nil, errors.Wrap(err, "unable to set linux container resources")
+	}
+	return &cloned, nil
+}

pkg/server/container_update_resources_linux_test.go

@@ -0,0 +1,162 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestUpdateOCILinuxResource(t *testing.T) {
+	oomscoreadj := new(int)
+	*oomscoreadj = -500
+	for desc, test := range map[string]struct {
+		spec      *runtimespec.Spec
+		resources *runtime.LinuxContainerResources
+		expected  *runtimespec.Spec
+		expectErr bool
+	}{
+		"should be able to update each resource": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(1111),
+							Quota:  proto.Int64(2222),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "2-3",
+						},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuPeriod:          6666,
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetCpus:         "4-5",
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(6666),
+							Cpus:   "4-5",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+		"should skip empty fields": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(1111),
+							Quota:  proto.Int64(2222),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "2-3",
+						},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+		"should be able to fill empty fields": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuPeriod:          6666,
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetCpus:         "4-5",
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(6666),
+							Cpus:   "4-5",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got, err := updateOCILinuxResource(context.Background(), test.spec, test.resources, false, false)
+		if test.expectErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+		assert.Equal(t, test.expected, got)
+	}
+}

pkg/server/container_update_resources_other.go

@@ -0,0 +1,44 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/cri/pkg/store/container"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (retRes *runtime.UpdateContainerResourcesResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	// Update resources in status update transaction, so that:
+	// 1) There won't be race condition with container start.
+	// 2) There won't be concurrent resource update to the same container.
+	if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		return status, nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to update resources")
+	}
+	return &runtime.UpdateContainerResourcesResponse{}, nil
+}

pkg/server/container_update_resources_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/errdefs"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+// TODO(windows): Figure out whether windows support this.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (*runtime.UpdateContainerResourcesResponse, error) {
+	return nil, errdefs.ErrNotImplemented
+}

pkg/server/events.go

@@ -0,0 +1,461 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd"
+	eventtypes "github.com/containerd/containerd/api/events"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/typeurl"
+	gogotypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/util/clock"
+
+	"github.com/containerd/cri/pkg/constants"
+	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
+	"github.com/containerd/cri/pkg/store"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
+)
+
+const (
+	backOffInitDuration        = 1 * time.Second
+	backOffMaxDuration         = 5 * time.Minute
+	backOffExpireCheckDuration = 1 * time.Second
+
+	// handleEventTimeout is the timeout for handling 1 event. Event monitor
+	// handles events in serial, if one event blocks the event monitor, no
+	// other events can be handled.
+	// Add a timeout for each event handling, events that timeout will be requeued and
+	// handled again in the future.
+	handleEventTimeout = 10 * time.Second
+
+	exitChannelSize = 1024
+)
+
+// eventMonitor monitors containerd event and updates internal state correspondingly.
+// TODO(random-liu): Handle event for each container in a separate goroutine.
+type eventMonitor struct {
+	c  *criService
+	ch <-chan *events.Envelope
+	// exitCh receives container/sandbox exit events from exit monitors.
+	exitCh  chan *eventtypes.TaskExit
+	errCh   <-chan error
+	ctx     context.Context
+	cancel  context.CancelFunc
+	backOff *backOff
+}
+
+type backOff struct {
+	queuePool map[string]*backOffQueue
+	// tickerMu is mutex used to protect the ticker.
+	tickerMu      sync.Mutex
+	ticker        *time.Ticker
+	minDuration   time.Duration
+	maxDuration   time.Duration
+	checkDuration time.Duration
+	clock         clock.Clock
+}
+
+type backOffQueue struct {
+	events     []interface{}
+	expireTime time.Time
+	duration   time.Duration
+	clock      clock.Clock
+}
+
+// Create new event monitor. New event monitor will start subscribing containerd event. All events
+// happen after it should be monitored.
+func newEventMonitor(c *criService) *eventMonitor {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &eventMonitor{
+		c:       c,
+		ctx:     ctx,
+		cancel:  cancel,
+		exitCh:  make(chan *eventtypes.TaskExit, exitChannelSize),
+		backOff: newBackOff(),
+	}
+}
+
+// subscribe starts to subscribe containerd events.
+func (em *eventMonitor) subscribe(subscriber events.Subscriber) {
+	// note: filters are any match, if you want any match but not in namespace foo
+	// then you have to manually filter namespace foo
+	filters := []string{
+		`topic=="/tasks/oom"`,
+		`topic~="/images/"`,
+	}
+	em.ch, em.errCh = subscriber.Subscribe(em.ctx, filters...)
+}
+
+// startExitMonitor starts an exit monitor for a given container/sandbox.
+func (em *eventMonitor) startExitMonitor(ctx context.Context, id string, pid uint32, exitCh <-chan containerd.ExitStatus) <-chan struct{} {
+	stopCh := make(chan struct{})
+	go func() {
+		defer close(stopCh)
+		select {
+		case exitRes := <-exitCh:
+			exitStatus, exitedAt, err := exitRes.Result()
+			if err != nil {
+				logrus.WithError(err).Errorf("Failed to get task exit status for %q", id)
+				exitStatus = unknownExitCode
+				exitedAt = time.Now()
+			}
+			em.exitCh <- &eventtypes.TaskExit{
+				ContainerID: id,
+				ID:          id,
+				Pid:         pid,
+				ExitStatus:  exitStatus,
+				ExitedAt:    exitedAt,
+			}
+		case <-ctx.Done():
+		}
+	}()
+	return stopCh
+}
+
+func convertEvent(e *gogotypes.Any) (string, interface{}, error) {
+	id := ""
+	evt, err := typeurl.UnmarshalAny(e)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to unmarshalany")
+	}
+
+	switch e := evt.(type) {
+	case *eventtypes.TaskOOM:
+		id = e.ContainerID
+	case *eventtypes.ImageCreate:
+		id = e.Name
+	case *eventtypes.ImageUpdate:
+		id = e.Name
+	case *eventtypes.ImageDelete:
+		id = e.Name
+	default:
+		return "", nil, errors.New("unsupported event")
+	}
+	return id, evt, nil
+}
+
+// start starts the event monitor which monitors and handles all subscribed events. It returns
+// an error channel for the caller to wait for stop errors from the event monitor.
+// start must be called after subscribe.
+func (em *eventMonitor) start() <-chan error {
+	errCh := make(chan error)
+	if em.ch == nil || em.errCh == nil {
+		panic("event channel is nil")
+	}
+	backOffCheckCh := em.backOff.start()
+	go func() {
+		defer close(errCh)
+		for {
+			select {
+			case e := <-em.exitCh:
+				logrus.Debugf("Received exit event %+v", e)
+				id := e.ID
+				if em.backOff.isInBackOff(id) {
+					logrus.Infof("Events for %q is in backoff, enqueue event %+v", id, e)
+					em.backOff.enBackOff(id, e)
+					break
+				}
+				if err := em.handleEvent(e); err != nil {
+					logrus.WithError(err).Errorf("Failed to handle exit event %+v for %s", e, id)
+					em.backOff.enBackOff(id, e)
+				}
+			case e := <-em.ch:
+				logrus.Debugf("Received containerd event timestamp - %v, namespace - %q, topic - %q", e.Timestamp, e.Namespace, e.Topic)
+				if e.Namespace != constants.K8sContainerdNamespace {
+					logrus.Debugf("Ignoring events in namespace - %q", e.Namespace)
+					break
+				}
+				id, evt, err := convertEvent(e.Event)
+				if err != nil {
+					logrus.WithError(err).Errorf("Failed to convert event %+v", e)
+					break
+				}
+				if em.backOff.isInBackOff(id) {
+					logrus.Infof("Events for %q is in backoff, enqueue event %+v", id, evt)
+					em.backOff.enBackOff(id, evt)
+					break
+				}
+				if err := em.handleEvent(evt); err != nil {
+					logrus.WithError(err).Errorf("Failed to handle event %+v for %s", evt, id)
+					em.backOff.enBackOff(id, evt)
+				}
+			case err := <-em.errCh:
+				// Close errCh in defer directly if there is no error.
+				if err != nil {
+					logrus.WithError(err).Errorf("Failed to handle event stream")
+					errCh <- err
+				}
+				return
+			case <-backOffCheckCh:
+				ids := em.backOff.getExpiredIDs()
+				for _, id := range ids {
+					queue := em.backOff.deBackOff(id)
+					for i, any := range queue.events {
+						if err := em.handleEvent(any); err != nil {
+							logrus.WithError(err).Errorf("Failed to handle backOff event %+v for %s", any, id)
+							em.backOff.reBackOff(id, queue.events[i:], queue.duration)
+							break
+						}
+					}
+				}
+			}
+		}
+	}()
+	return errCh
+}
+
+// stop stops the event monitor. It will close the event channel.
+// Once event monitor is stopped, it can't be started.
+func (em *eventMonitor) stop() {
+	em.backOff.stop()
+	em.cancel()
+}
+
+// handleEvent handles a containerd event.
+func (em *eventMonitor) handleEvent(any interface{}) error {
+	ctx := ctrdutil.NamespacedContext()
+	ctx, cancel := context.WithTimeout(ctx, handleEventTimeout)
+	defer cancel()
+
+	switch e := any.(type) {
+	case *eventtypes.TaskExit:
+		logrus.Infof("TaskExit event %+v", e)
+		// Use ID instead of ContainerID to rule out TaskExit event for exec.
+		cntr, err := em.c.containerStore.Get(e.ID)
+		if err == nil {
+			if err := handleContainerExit(ctx, e, cntr); err != nil {
+				return errors.Wrap(err, "failed to handle container TaskExit event")
+			}
+			return nil
+		} else if err != store.ErrNotExist {
+			return errors.Wrap(err, "can't find container for TaskExit event")
+		}
+		sb, err := em.c.sandboxStore.Get(e.ID)
+		if err == nil {
+			if err := handleSandboxExit(ctx, e, sb); err != nil {
+				return errors.Wrap(err, "failed to handle sandbox TaskExit event")
+			}
+			return nil
+		} else if err != store.ErrNotExist {
+			return errors.Wrap(err, "can't find sandbox for TaskExit event")
+		}
+		return nil
+	case *eventtypes.TaskOOM:
+		logrus.Infof("TaskOOM event %+v", e)
+		// For TaskOOM, we only care which container it belongs to.
+		cntr, err := em.c.containerStore.Get(e.ContainerID)
+		if err != nil {
+			if err != store.ErrNotExist {
+				return errors.Wrap(err, "can't find container for TaskOOM event")
+			}
+			return nil
+		}
+		err = cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+			status.Reason = oomExitReason
+			return status, nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "failed to update container status for TaskOOM event")
+		}
+	case *eventtypes.ImageCreate:
+		logrus.Infof("ImageCreate event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	case *eventtypes.ImageUpdate:
+		logrus.Infof("ImageUpdate event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	case *eventtypes.ImageDelete:
+		logrus.Infof("ImageDelete event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	}
+
+	return nil
+}
+
+// handleContainerExit handles TaskExit event for container.
+func handleContainerExit(ctx context.Context, e *eventtypes.TaskExit, cntr containerstore.Container) error {
+	// Attach container IO so that `Delete` could cleanup the stream properly.
+	task, err := cntr.Container.Task(ctx,
+		func(*containerdio.FIFOSet) (containerdio.IO, error) {
+			// We can't directly return cntr.IO here, because
+			// even if cntr.IO is nil, the cio.IO interface
+			// is not.
+			// See https://tour.golang.org/methods/12:
+			//   Note that an interface value that holds a nil
+			//   concrete value is itself non-nil.
+			if cntr.IO != nil {
+				return cntr.IO, nil
+			}
+			return nil, nil
+		},
+	)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to load task for container")
+		}
+	} else {
+		// TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker
+		if _, err = task.Delete(ctx, WithNRISandboxDelete(cntr.SandboxID), containerd.WithProcessKill); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrap(err, "failed to stop container")
+			}
+			// Move on to make sure container status is updated.
+		}
+	}
+	err = cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+		// If FinishedAt has been set (e.g. with start failure), keep as
+		// it is.
+		if status.FinishedAt != 0 {
+			return status, nil
+		}
+		status.Pid = 0
+		status.FinishedAt = e.ExitedAt.UnixNano()
+		status.ExitCode = int32(e.ExitStatus)
+		// Unknown state can only transit to EXITED state, so we need
+		// to handle unknown state here.
+		if status.Unknown {
+			logrus.Debugf("Container %q transited from UNKNOWN to EXITED", cntr.ID)
+			status.Unknown = false
+		}
+		return status, nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to update container state")
+	}
+	// Using channel to propagate the information of container stop
+	cntr.Stop()
+	return nil
+}
+
+// handleSandboxExit handles TaskExit event for sandbox.
+func handleSandboxExit(ctx context.Context, e *eventtypes.TaskExit, sb sandboxstore.Sandbox) error {
+	// No stream attached to sandbox container.
+	task, err := sb.Container.Task(ctx, nil)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrap(err, "failed to load task for sandbox")
+		}
+	} else {
+		// TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker
+		if _, err = task.Delete(ctx, WithNRISandboxDelete(sb.ID), containerd.WithProcessKill); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrap(err, "failed to stop sandbox")
+			}
+			// Move on to make sure container status is updated.
+		}
+	}
+	err = sb.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
+		status.State = sandboxstore.StateNotReady
+		status.Pid = 0
+		return status, nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to update sandbox state")
+	}
+	// Using channel to propagate the information of sandbox stop
+	sb.Stop()
+	return nil
+}
+
+func newBackOff() *backOff {
+	return &backOff{
+		queuePool:     map[string]*backOffQueue{},
+		minDuration:   backOffInitDuration,
+		maxDuration:   backOffMaxDuration,
+		checkDuration: backOffExpireCheckDuration,
+		clock:         clock.RealClock{},
+	}
+}
+
+func (b *backOff) getExpiredIDs() []string {
+	var ids []string
+	for id, q := range b.queuePool {
+		if q.isExpire() {
+			ids = append(ids, id)
+		}
+	}
+	return ids
+}
+
+func (b *backOff) isInBackOff(key string) bool {
+	if _, ok := b.queuePool[key]; ok {
+		return true
+	}
+	return false
+}
+
+// enBackOff start to backOff and put event to the tail of queue
+func (b *backOff) enBackOff(key string, evt interface{}) {
+	if queue, ok := b.queuePool[key]; ok {
+		queue.events = append(queue.events, evt)
+		return
+	}
+	b.queuePool[key] = newBackOffQueue([]interface{}{evt}, b.minDuration, b.clock)
+}
+
+// enBackOff get out the whole queue
+func (b *backOff) deBackOff(key string) *backOffQueue {
+	queue := b.queuePool[key]
+	delete(b.queuePool, key)
+	return queue
+}
+
+// enBackOff start to backOff again and put events to the queue
+func (b *backOff) reBackOff(key string, events []interface{}, oldDuration time.Duration) {
+	duration := 2 * oldDuration
+	if duration > b.maxDuration {
+		duration = b.maxDuration
+	}
+	b.queuePool[key] = newBackOffQueue(events, duration, b.clock)
+}
+
+func (b *backOff) start() <-chan time.Time {
+	b.tickerMu.Lock()
+	defer b.tickerMu.Unlock()
+	b.ticker = time.NewTicker(b.checkDuration)
+	return b.ticker.C
+}
+
+func (b *backOff) stop() {
+	b.tickerMu.Lock()
+	defer b.tickerMu.Unlock()
+	if b.ticker != nil {
+		b.ticker.Stop()
+	}
+}
+
+func newBackOffQueue(events []interface{}, init time.Duration, c clock.Clock) *backOffQueue {
+	return &backOffQueue{
+		events:     events,
+		duration:   init,
+		expireTime: c.Now().Add(init),
+		clock:      c,
+	}
+}
+
+func (q *backOffQueue) isExpire() bool {
+	// return time.Now >= expireTime
+	return !q.clock.Now().Before(q.expireTime)
+}

pkg/server/events_test.go

@@ -0,0 +1,134 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	eventtypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/typeurl"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/clock"
+)
+
+// TestBackOff tests the logic of backOff struct.
+func TestBackOff(t *testing.T) {
+	testStartTime := time.Now()
+	testClock := clock.NewFakeClock(testStartTime)
+	inputQueues := map[string]*backOffQueue{
+		"container1": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+			},
+		},
+		"container2": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+			},
+		},
+	}
+	expectedQueues := map[string]*backOffQueue{
+		"container2": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+			},
+			expireTime: testClock.Now().Add(backOffInitDuration),
+			duration:   backOffInitDuration,
+			clock:      testClock,
+		},
+		"container1": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+			},
+			expireTime: testClock.Now().Add(backOffInitDuration),
+			duration:   backOffInitDuration,
+			clock:      testClock,
+		},
+	}
+
+	t.Logf("Should be able to backOff a event")
+	actual := newBackOff()
+	actual.clock = testClock
+	for k, queue := range inputQueues {
+		for _, event := range queue.events {
+			actual.enBackOff(k, event)
+		}
+	}
+	assert.Equal(t, actual.queuePool, expectedQueues)
+
+	t.Logf("Should be able to check if the container is in backOff state")
+	for k, queue := range inputQueues {
+		for _, e := range queue.events {
+			any, err := typeurl.MarshalAny(e)
+			assert.NoError(t, err)
+			key, _, err := convertEvent(any)
+			assert.NoError(t, err)
+			assert.Equal(t, k, key)
+			assert.Equal(t, actual.isInBackOff(key), true)
+		}
+	}
+
+	t.Logf("Should be able to check that a container isn't in backOff state")
+	notExistKey := "containerNotExist"
+	assert.Equal(t, actual.isInBackOff(notExistKey), false)
+
+	t.Logf("No containers should be expired")
+	assert.Empty(t, actual.getExpiredIDs())
+
+	t.Logf("Should be able to get all keys which are expired for backOff")
+	testClock.Sleep(backOffInitDuration)
+	actKeyList := actual.getExpiredIDs()
+	assert.Equal(t, len(inputQueues), len(actKeyList))
+	for k := range inputQueues {
+		assert.Contains(t, actKeyList, k)
+	}
+
+	t.Logf("Should be able to get out all backOff events")
+	doneQueues := map[string]*backOffQueue{}
+	for k := range inputQueues {
+		actQueue := actual.deBackOff(k)
+		doneQueues[k] = actQueue
+		assert.Equal(t, actQueue, expectedQueues[k])
+	}
+
+	t.Logf("Should not get out the event again after having got out the backOff event")
+	for k := range inputQueues {
+		var expect *backOffQueue
+		actQueue := actual.deBackOff(k)
+		assert.Equal(t, actQueue, expect)
+	}
+
+	t.Logf("Should be able to reBackOff")
+	for k, queue := range doneQueues {
+		failEventIndex := 1
+		events := queue.events[failEventIndex:]
+		actual.reBackOff(k, events, queue.duration)
+		actQueue := actual.deBackOff(k)
+		expQueue := &backOffQueue{
+			events:     events,
+			expireTime: testClock.Now().Add(2 * queue.duration),
+			duration:   2 * queue.duration,
+			clock:      testClock,
+		}
+		assert.Equal(t, actQueue, expQueue)
+	}
+}

pkg/server/helpers.go

@@ -0,0 +1,390 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/typeurl"
+	imagedigest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	runtimeoptions "github.com/containerd/cri/pkg/api/runtimeoptions/v1"
+	criconfig "github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/store"
+	containerstore "github.com/containerd/cri/pkg/store/container"
+	imagestore "github.com/containerd/cri/pkg/store/image"
+	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
+)
+
+const (
+	// errorStartReason is the exit reason when fails to start container.
+	errorStartReason = "StartError"
+	// errorStartExitCode is the exit code when fails to start container.
+	// 128 is the same with Docker's behavior.
+	// TODO(windows): Figure out what should be used for windows.
+	errorStartExitCode = 128
+	// completeExitReason is the exit reason when container exits with code 0.
+	completeExitReason = "Completed"
+	// errorExitReason is the exit reason when container exits with code non-zero.
+	errorExitReason = "Error"
+	// oomExitReason is the exit reason when process in container is oom killed.
+	oomExitReason = "OOMKilled"
+
+	// sandboxesDir contains all sandbox root. A sandbox root is the running
+	// directory of the sandbox, all files created for the sandbox will be
+	// placed under this directory.
+	sandboxesDir = "sandboxes"
+	// containersDir contains all container root.
+	containersDir = "containers"
+	// Delimiter used to construct container/sandbox names.
+	nameDelimiter = "_"
+
+	// criContainerdPrefix is common prefix for cri-containerd
+	criContainerdPrefix = "io.cri-containerd"
+	// containerKindLabel is a label key indicating container is sandbox container or application container
+	containerKindLabel = criContainerdPrefix + ".kind"
+	// containerKindSandbox is a label value indicating container is sandbox container
+	containerKindSandbox = "sandbox"
+	// containerKindContainer is a label value indicating container is application container
+	containerKindContainer = "container"
+	// imageLabelKey is the label key indicating the image is managed by cri plugin.
+	imageLabelKey = criContainerdPrefix + ".image"
+	// imageLabelValue is the label value indicating the image is managed by cri plugin.
+	imageLabelValue = "managed"
+	// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
+	sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
+	// containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest
+	containerMetadataExtension = criContainerdPrefix + ".container.metadata"
+
+	// defaultIfName is the default network interface for the pods
+	defaultIfName = "eth0"
+
+	// runtimeRunhcsV1 is the runtime type for runhcs.
+	runtimeRunhcsV1 = "io.containerd.runhcs.v1"
+)
+
+// makeSandboxName generates sandbox name from sandbox metadata. The name
+// generated is unique as long as sandbox metadata is unique.
+func makeSandboxName(s *runtime.PodSandboxMetadata) string {
+	return strings.Join([]string{
+		s.Name,                       // 0
+		s.Namespace,                  // 1
+		s.Uid,                        // 2
+		fmt.Sprintf("%d", s.Attempt), // 3
+	}, nameDelimiter)
+}
+
+// makeContainerName generates container name from sandbox and container metadata.
+// The name generated is unique as long as the sandbox container combination is
+// unique.
+func makeContainerName(c *runtime.ContainerMetadata, s *runtime.PodSandboxMetadata) string {
+	return strings.Join([]string{
+		c.Name,                       // 0
+		s.Name,                       // 1: pod name
+		s.Namespace,                  // 2: pod namespace
+		s.Uid,                        // 3: pod uid
+		fmt.Sprintf("%d", c.Attempt), // 4
+	}, nameDelimiter)
+}
+
+// getSandboxRootDir returns the root directory for managing sandbox files,
+// e.g. hosts files.
+func (c *criService) getSandboxRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, sandboxesDir, id)
+}
+
+// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
+// e.g. named pipes.
+func (c *criService) getVolatileSandboxRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, sandboxesDir, id)
+}
+
+// getContainerRootDir returns the root directory for managing container files,
+// e.g. state checkpoint.
+func (c *criService) getContainerRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, containersDir, id)
+}
+
+// getVolatileContainerRootDir returns the root directory for managing volatile container files,
+// e.g. named pipes.
+func (c *criService) getVolatileContainerRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, containersDir, id)
+}
+
+// criContainerStateToString formats CRI container state to string.
+func criContainerStateToString(state runtime.ContainerState) string {
+	return runtime.ContainerState_name[int32(state)]
+}
+
+// getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference.
+func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) {
+	var repoTag, repoDigest string
+	if _, ok := namedRef.(docker.NamedTagged); ok {
+		repoTag = namedRef.String()
+	}
+	if _, ok := namedRef.(docker.Canonical); ok {
+		repoDigest = namedRef.String()
+	} else if !schema1 {
+		// digest is not actual repo digest for schema1 image.
+		repoDigest = namedRef.Name() + "@" + digest.String()
+	}
+	return repoDigest, repoTag
+}
+
+// localResolve resolves image reference locally and returns corresponding image metadata. It
+// returns store.ErrNotExist if the reference doesn't exist.
+func (c *criService) localResolve(refOrID string) (imagestore.Image, error) {
+	getImageID := func(refOrId string) string {
+		if _, err := imagedigest.Parse(refOrID); err == nil {
+			return refOrID
+		}
+		return func(ref string) string {
+			// ref is not image id, try to resolve it locally.
+			// TODO(random-liu): Handle this error better for debugging.
+			normalized, err := docker.ParseDockerRef(ref)
+			if err != nil {
+				return ""
+			}
+			id, err := c.imageStore.Resolve(normalized.String())
+			if err != nil {
+				return ""
+			}
+			return id
+		}(refOrID)
+	}
+
+	imageID := getImageID(refOrID)
+	if imageID == "" {
+		// Try to treat ref as imageID
+		imageID = refOrID
+	}
+	return c.imageStore.Get(imageID)
+}
+
+// toContainerdImage converts an image object in image store to containerd image handler.
+func (c *criService) toContainerdImage(ctx context.Context, image imagestore.Image) (containerd.Image, error) {
+	// image should always have at least one reference.
+	if len(image.References) == 0 {
+		return nil, errors.Errorf("invalid image with no reference %q", image.ID)
+	}
+	return c.client.GetImage(ctx, image.References[0])
+}
+
+// getUserFromImage gets uid or user name of the image user.
+// If user is numeric, it will be treated as uid; or else, it is treated as user name.
+func getUserFromImage(user string) (*int64, string) {
+	// return both empty if user is not specified in the image.
+	if user == "" {
+		return nil, ""
+	}
+	// split instances where the id may contain user:group
+	user = strings.Split(user, ":")[0]
+	// user could be either uid or user name. Try to interpret as numeric uid.
+	uid, err := strconv.ParseInt(user, 10, 64)
+	if err != nil {
+		// If user is non numeric, assume it's user name.
+		return nil, user
+	}
+	// If user is a numeric uid.
+	return &uid, ""
+}
+
+// ensureImageExists returns corresponding metadata of the image reference, if image is not
+// pulled yet, the function will pull the image.
+func (c *criService) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig) (*imagestore.Image, error) {
+	image, err := c.localResolve(ref)
+	if err != nil && err != store.ErrNotExist {
+		return nil, errors.Wrapf(err, "failed to get image %q", ref)
+	}
+	if err == nil {
+		return &image, nil
+	}
+	// Pull image to ensure the image exists
+	resp, err := c.PullImage(ctx, &runtime.PullImageRequest{Image: &runtime.ImageSpec{Image: ref}, SandboxConfig: config})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to pull image %q", ref)
+	}
+	imageID := resp.GetImageRef()
+	newImage, err := c.imageStore.Get(imageID)
+	if err != nil {
+		// It's still possible that someone removed the image right after it is pulled.
+		return nil, errors.Wrapf(err, "failed to get image %q after pulling", imageID)
+	}
+	return &newImage, nil
+}
+
+// isInCRIMounts checks whether a destination is in CRI mount list.
+func isInCRIMounts(dst string, mounts []*runtime.Mount) bool {
+	for _, m := range mounts {
+		if filepath.Clean(m.ContainerPath) == filepath.Clean(dst) {
+			return true
+		}
+	}
+	return false
+}
+
+// filterLabel returns a label filter. Use `%q` here because containerd
+// filter needs extra quote to work properly.
+func filterLabel(k, v string) string {
+	return fmt.Sprintf("labels.%q==%q", k, v)
+}
+
+// buildLabel builds the labels from config to be passed to containerd
+func buildLabels(configLabels map[string]string, containerType string) map[string]string {
+	labels := make(map[string]string)
+	for k, v := range configLabels {
+		labels[k] = v
+	}
+	labels[containerKindLabel] = containerType
+	return labels
+}
+
+// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
+func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
+	return &runtime.AuthConfig{
+		Username:      a.Username,
+		Password:      a.Password,
+		Auth:          a.Auth,
+		IdentityToken: a.IdentityToken,
+	}
+}
+
+// parseImageReferences parses a list of arbitrary image references and returns
+// the repotags and repodigests
+func parseImageReferences(refs []string) ([]string, []string) {
+	var tags, digests []string
+	for _, ref := range refs {
+		parsed, err := docker.ParseAnyReference(ref)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsed.(docker.Canonical); ok {
+			digests = append(digests, parsed.String())
+		} else if _, ok := parsed.(docker.Tagged); ok {
+			tags = append(tags, parsed.String())
+		}
+	}
+	return tags, digests
+}
+
+// generateRuntimeOptions generates runtime options from cri plugin config.
+func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
+	if r.Options == nil {
+		if r.Type != plugin.RuntimeLinuxV1 {
+			return nil, nil
+		}
+		// This is a legacy config, generate runctypes.RuncOptions.
+		return &runctypes.RuncOptions{
+			Runtime:       r.Engine,
+			RuntimeRoot:   r.Root,
+			SystemdCgroup: c.SystemdCgroup,
+		}, nil
+	}
+	options := getRuntimeOptionsType(r.Type)
+	if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
+		return nil, err
+	}
+	return options, nil
+}
+
+// getRuntimeOptionsType gets empty runtime options by the runtime type name.
+func getRuntimeOptionsType(t string) interface{} {
+	switch t {
+	case plugin.RuntimeRuncV1:
+		fallthrough
+	case plugin.RuntimeRuncV2:
+		return &runcoptions.Options{}
+	case plugin.RuntimeLinuxV1:
+		return &runctypes.RuncOptions{}
+	case runtimeRunhcsV1:
+		return &runhcsoptions.Options{}
+	default:
+		return &runtimeoptions.Options{}
+	}
+}
+
+// getRuntimeOptions get runtime options from container metadata.
+func getRuntimeOptions(c containers.Container) (interface{}, error) {
+	if c.Runtime.Options == nil {
+		return nil, nil
+	}
+	opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
+	if err != nil {
+		return nil, err
+	}
+	return opts, nil
+}
+
+const (
+	// unknownExitCode is the exit code when exit reason is unknown.
+	unknownExitCode = 255
+	// unknownExitReason is the exit reason when exit reason is unknown.
+	unknownExitReason = "Unknown"
+)
+
+// unknownContainerStatus returns the default container status when its status is unknown.
+func unknownContainerStatus() containerstore.Status {
+	return containerstore.Status{
+		CreatedAt:  0,
+		StartedAt:  0,
+		FinishedAt: 0,
+		ExitCode:   unknownExitCode,
+		Reason:     unknownExitReason,
+		Unknown:    true,
+	}
+}
+
+// unknownSandboxStatus returns the default sandbox status when its status is unknown.
+func unknownSandboxStatus() sandboxstore.Status {
+	return sandboxstore.Status{
+		State: sandboxstore.StateUnknown,
+	}
+}
+
+// getPassthroughAnnotations filters requested pod annotations by comparing
+// against permitted annotations for the given runtime.
+func getPassthroughAnnotations(podAnnotations map[string]string,
+	runtimePodAnnotations []string) (passthroughAnnotations map[string]string) {
+	passthroughAnnotations = make(map[string]string)
+
+	for podAnnotationKey, podAnnotationValue := range podAnnotations {
+		for _, pattern := range runtimePodAnnotations {
+			// Use path.Match instead of filepath.Match here.
+			// filepath.Match treated `\\` as path separator
+			// on windows, which is not what we want.
+			if ok, _ := path.Match(pattern, podAnnotationKey); ok {
+				passthroughAnnotations[podAnnotationKey] = podAnnotationValue
+			}
+		}
+	}
+	return passthroughAnnotations
+}

pkg/server/helpers_linux.go

@@ -0,0 +1,290 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/cri/pkg/seccomp"
+	"github.com/containerd/cri/pkg/seutil"
+	runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+const (
+	// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
+	defaultSandboxOOMAdj = -998
+	// defaultShmSize is the default size of the sandbox shm.
+	defaultShmSize = int64(1024 * 1024 * 64)
+	// relativeRootfsPath is the rootfs path relative to bundle path.
+	relativeRootfsPath = "rootfs"
+	// According to http://man7.org/linux/man-pages/man5/resolv.conf.5.html:
+	// "The search list is currently limited to six domains with a total of 256 characters."
+	maxDNSSearches = 6
+	// devShm is the default path of /dev/shm.
+	devShm = "/dev/shm"
+	// etcHosts is the default path of /etc/hosts file.
+	etcHosts = "/etc/hosts"
+	// etcHostname is the default path of /etc/hostname file.
+	etcHostname = "/etc/hostname"
+	// resolvConfPath is the abs path of resolv.conf on host or container.
+	resolvConfPath = "/etc/resolv.conf"
+	// hostnameEnv is the key for HOSTNAME env.
+	hostnameEnv = "HOSTNAME"
+)
+
+// getCgroupsPath generates container cgroups path.
+func getCgroupsPath(cgroupsParent, id string) string {
+	base := path.Base(cgroupsParent)
+	if strings.HasSuffix(base, ".slice") {
+		// For a.slice/b.slice/c.slice, base is c.slice.
+		// runc systemd cgroup path format is "slice:prefix:name".
+		return strings.Join([]string{base, "cri-containerd", id}, ":")
+	}
+	return filepath.Join(cgroupsParent, id)
+}
+
+// getSandboxHostname returns the hostname file path inside the sandbox root directory.
+func (c *criService) getSandboxHostname(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hostname")
+}
+
+// getSandboxHosts returns the hosts file path inside the sandbox root directory.
+func (c *criService) getSandboxHosts(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hosts")
+}
+
+// getResolvPath returns resolv.conf filepath for specified sandbox.
+func (c *criService) getResolvPath(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
+}
+
+// getSandboxDevShm returns the shm file path inside the sandbox root directory.
+func (c *criService) getSandboxDevShm(id string) string {
+	return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
+}
+
+func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
+	var labels []string
+
+	if selinuxOptions == nil {
+		return nil, nil
+	}
+	if err := checkSelinuxLevel(selinuxOptions.Level); err != nil {
+		return nil, err
+	}
+	if selinuxOptions.User != "" {
+		labels = append(labels, "user:"+selinuxOptions.User)
+	}
+	if selinuxOptions.Role != "" {
+		labels = append(labels, "role:"+selinuxOptions.Role)
+	}
+	if selinuxOptions.Type != "" {
+		labels = append(labels, "type:"+selinuxOptions.Type)
+	}
+	if selinuxOptions.Level != "" {
+		labels = append(labels, "level:"+selinuxOptions.Level)
+	}
+
+	return labels, nil
+}
+
+func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
+	labels, err := toLabel(selinuxOpts)
+	if err != nil {
+		return "", "", err
+	}
+	return label.InitLabels(labels)
+}
+
+func checkSelinuxLevel(level string) error {
+	if len(level) == 0 {
+		return nil
+	}
+
+	matched, err := regexp.MatchString(`^s\d(-s\d)??(:c\d{1,4}(\.c\d{1,4})?(,c\d{1,4}(\.c\d{1,4})?)*)?$`, level)
+	if err != nil {
+		return errors.Wrapf(err, "the format of 'level' %q is not correct", level)
+	}
+	if !matched {
+		return fmt.Errorf("the format of 'level' %q is not correct", level)
+	}
+	return nil
+}
+
+func (c *criService) apparmorEnabled() bool {
+	return runcapparmor.IsEnabled() && !c.config.DisableApparmor
+}
+
+func (c *criService) seccompEnabled() bool {
+	return seccomp.IsEnabled()
+}
+
+// openLogFile opens/creates a container log file.
+func openLogFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+// unmountRecursive unmounts the target and all mounts underneath, starting with
+// the deepest mount first.
+func unmountRecursive(ctx context.Context, target string) error {
+	mounts, err := mount.Self()
+	if err != nil {
+		return err
+	}
+
+	var toUnmount []string
+	for _, m := range mounts {
+		p, err := filepath.Rel(target, m.Mountpoint)
+		if err != nil {
+			return err
+		}
+		if !strings.HasPrefix(p, "..") {
+			toUnmount = append(toUnmount, m.Mountpoint)
+		}
+	}
+
+	// Make the deepest mount be first
+	sort.Slice(toUnmount, func(i, j int) bool {
+		return len(toUnmount[i]) > len(toUnmount[j])
+	})
+
+	for i, mountPath := range toUnmount {
+		if err := mount.UnmountAll(mountPath, unix.MNT_DETACH); err != nil {
+			if i == len(toUnmount)-1 { // last mount
+				return err
+			}
+			// This is some submount, we can ignore this error for now, the final unmount will fail if this is a real problem
+			log.G(ctx).WithError(err).Debugf("failed to unmount submount %s", mountPath)
+		}
+	}
+	return nil
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+//
+// Because of the way `os.Remove` (and by extension `os.RemoveAll`) works, there
+// can be a race between reading directory entries and then actually attempting
+// to remove everything in the directory.
+// These types of errors do not need to be returned since it's ok for the dir to
+// be gone we can just retry the remove operation.
+//
+// This should not return a `os.ErrNotExist` kind of error under any circumstances
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	notExistErr := make(map[string]bool)
+
+	// track retries
+	exitOnErr := make(map[string]int)
+	maxRetry := 50
+
+	// Attempt to unmount anything beneath this dir first.
+	if err := unmountRecursive(ctx, dir); err != nil {
+		log.G(ctx).WithError(err).Debugf("failed to do initial unmount of %s", dir)
+	}
+
+	for {
+		err := os.RemoveAll(dir)
+		if err == nil {
+			return nil
+		}
+
+		pe, ok := err.(*os.PathError)
+		if !ok {
+			return err
+		}
+
+		if os.IsNotExist(err) {
+			if notExistErr[pe.Path] {
+				return err
+			}
+			notExistErr[pe.Path] = true
+
+			// There is a race where some subdir can be removed but after the
+			// parent dir entries have been read.
+			// So the path could be from `os.Remove(subdir)`
+			// If the reported non-existent path is not the passed in `dir` we
+			// should just retry, but otherwise return with no error.
+			if pe.Path == dir {
+				return nil
+			}
+			continue
+		}
+
+		if pe.Err != syscall.EBUSY {
+			return err
+		}
+		if e := mount.Unmount(pe.Path, unix.MNT_DETACH); e != nil {
+			return errors.Wrapf(e, "error while removing %s", dir)
+		}
+
+		if exitOnErr[pe.Path] == maxRetry {
+			return err
+		}
+		exitOnErr[pe.Path]++
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+var vmbasedRuntimes = []string{
+	"io.containerd.kata",
+}
+
+func isVMBasedRuntime(runtimeType string) bool {
+	for _, rt := range vmbasedRuntimes {
+		if strings.Contains(runtimeType, rt) {
+			return true
+		}
+	}
+	return false
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	if !isVMBasedRuntime(runtimeType) {
+		return nil
+	}
+	l, err := getKVMLabel(spec.Process.SelinuxLabel)
+	if err != nil {
+		return errors.Wrap(err, "failed to get selinux kvm label")
+	}
+	spec.Process.SelinuxLabel = l
+	return nil
+}
+
+func getKVMLabel(l string) (string, error) {
+	if !seutil.HasType("container_kvm_t") {
+		return "", nil
+	}
+	return seutil.ChangeToKVM(l)
+}

pkg/server/helpers_linux_test.go

@@ -0,0 +1,106 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+	"golang.org/x/sys/unix"
+)
+
+func TestGetCgroupsPath(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		cgroupsParent string
+		expected      string
+	}{
+		"should support regular cgroup path": {
+			cgroupsParent: "/a/b",
+			expected:      "/a/b/test-id",
+		},
+		"should support systemd cgroup path": {
+			cgroupsParent: "/a.slice/b.slice",
+			expected:      "b.slice:cri-containerd:test-id",
+		},
+		"should support tailing slash for regular cgroup path": {
+			cgroupsParent: "/a/b/",
+			expected:      "/a/b/test-id",
+		},
+		"should support tailing slash for systemd cgroup path": {
+			cgroupsParent: "/a.slice/b.slice/",
+			expected:      "b.slice:cri-containerd:test-id",
+		},
+		"should treat root cgroup as regular cgroup path": {
+			cgroupsParent: "/",
+			expected:      "/test-id",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got := getCgroupsPath(test.cgroupsParent, testID)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestEnsureRemoveAllWithMount(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("skipping test that requires root")
+	}
+
+	dir1, err := ioutil.TempDir("", "test-ensure-removeall-with-dir1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir2, err := ioutil.TempDir("", "test-ensure-removeall-with-dir2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir2)
+
+	bindDir := filepath.Join(dir1, "bind")
+	if err := os.MkdirAll(bindDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := unix.Mount(dir2, bindDir, "none", unix.MS_BIND, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		err = ensureRemoveAll(context.Background(), dir1)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("timeout waiting for EnsureRemoveAll to finish")
+	}
+
+	if _, err := os.Stat(dir1); !os.IsNotExist(err) {
+		t.Fatalf("expected %q to not exist", dir1)
+	}
+}

pkg/server/helpers_other.go

@@ -0,0 +1,43 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// openLogFile opens/creates a container log file.
+func openLogFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/server/helpers_selinux_test.go

@@ -0,0 +1,159 @@
+// +build selinux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestInitSelinuxOpts(t *testing.T) {
+	if !selinux.GetEnabled() {
+		t.Skip("selinux is not enabled")
+	}
+
+	for desc, test := range map[string]struct {
+		selinuxOpt   *runtime.SELinuxOption
+		processLabel string
+		mountLabel   string
+		expectErr    bool
+	}{
+		"Should return empty strings for processLabel and mountLabel when selinuxOpt is nil": {
+			selinuxOpt:   nil,
+			processLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}",
+			mountLabel:   ".*:c[0-9]{1,3},c[0-9]{1,3}",
+		},
+		"Should overlay fields on processLabel when selinuxOpt has been initialized partially": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "",
+				Role:  "user_r",
+				Type:  "",
+				Level: "s0:c1,c2",
+			},
+			processLabel: "system_u:user_r:(container_file_t|svirt_lxc_net_t):s0:c1,c2",
+			mountLabel:   "system_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
+		},
+		"Should be resolved correctly when selinuxOpt has been initialized completely": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "s0:c1,c2",
+			},
+			processLabel: "user_u:user_r:user_t:s0:c1,c2",
+			mountLabel:   "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
+		},
+		"Should be resolved correctly when selinuxOpt has been initialized with level=''": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "",
+			},
+			processLabel: "user_u:user_r:user_t:s0:c[0-9]{1,3},c[0-9]{1,3}",
+			mountLabel:   "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0",
+		},
+		"Should return error when the format of 'level' is not correct": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "s0,c1,c2",
+			},
+			expectErr: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			processLabel, mountLabel, err := initLabelsFromOpt(test.selinuxOpt)
+			if test.expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.Regexp(t, test.processLabel, processLabel)
+				assert.Regexp(t, test.mountLabel, mountLabel)
+			}
+		})
+	}
+}
+
+func TestCheckSelinuxLevel(t *testing.T) {
+	for desc, test := range map[string]struct {
+		level         string
+		expectNoMatch bool
+	}{
+		"s0": {
+			level: "s0",
+		},
+		"s0-s0": {
+			level: "s0-s0",
+		},
+		"s0:c0": {
+			level: "s0:c0",
+		},
+		"s0:c0.c3": {
+			level: "s0:c0.c3",
+		},
+		"s0:c0,c3": {
+			level: "s0:c0,c3",
+		},
+		"s0-s0:c0,c3": {
+			level: "s0-s0:c0,c3",
+		},
+		"s0-s0:c0,c3.c6": {
+			level: "s0-s0:c0,c3.c6",
+		},
+		"s0-s0:c0,c3.c6,c8.c10": {
+			level: "s0-s0:c0,c3.c6,c8.c10",
+		},
+		"s0-s0:c0,c3.c6,c8,c10": {
+			level: "s0-s0:c0,c3.c6",
+		},
+		"s0,c0,c3": {
+			level:         "s0,c0,c3",
+			expectNoMatch: true,
+		},
+		"s0:c0.c3.c6": {
+			level:         "s0:c0.c3.c6",
+			expectNoMatch: true,
+		},
+		"s0-s0,c0,c3": {
+			level:         "s0-s0,c0,c3",
+			expectNoMatch: true,
+		},
+		"s0-s0:c0.c3.c6": {
+			level:         "s0-s0:c0.c3.c6",
+			expectNoMatch: true,
+		},
+		"s0-s0:c0,c3.c6.c8": {
+			level:         "s0-s0:c0,c3.c6.c8",
+			expectNoMatch: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			err := checkSelinuxLevel(test.level)
+			if test.expectNoMatch {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

pkg/server/helpers_test.go

@@ -0,0 +1,498 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	imagedigest "github.com/opencontainers/go-digest"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+	"github.com/containerd/cri/pkg/store"
+	imagestore "github.com/containerd/cri/pkg/store/image"
+)
+
+// TestGetUserFromImage tests the logic of getting image uid or user name of image user.
+func TestGetUserFromImage(t *testing.T) {
+	newI64 := func(i int64) *int64 { return &i }
+	for c, test := range map[string]struct {
+		user string
+		uid  *int64
+		name string
+	}{
+		"no gid": {
+			user: "0",
+			uid:  newI64(0),
+		},
+		"uid/gid": {
+			user: "0:1",
+			uid:  newI64(0),
+		},
+		"empty user": {
+			user: "",
+		},
+		"multiple spearators": {
+			user: "1:2:3",
+			uid:  newI64(1),
+		},
+		"root username": {
+			user: "root:root",
+			name: "root",
+		},
+		"username": {
+			user: "test:test",
+			name: "test",
+		},
+	} {
+		t.Logf("TestCase - %q", c)
+		actualUID, actualName := getUserFromImage(test.user)
+		assert.Equal(t, test.uid, actualUID)
+		assert.Equal(t, test.name, actualName)
+	}
+}
+
+func TestGetRepoDigestAndTag(t *testing.T) {
+	digest := imagedigest.Digest("sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582")
+	for desc, test := range map[string]struct {
+		ref                string
+		schema1            bool
+		expectedRepoDigest string
+		expectedRepoTag    string
+	}{
+		"repo tag should be empty if original ref has no tag": {
+			ref:                "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+		},
+		"repo tag should not be empty if original ref has tag": {
+			ref:                "gcr.io/library/busybox:latest",
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should be empty if original ref is schema1 and has no digest": {
+			ref:                "gcr.io/library/busybox:latest",
+			schema1:            true,
+			expectedRepoDigest: "",
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should not be empty if orignal ref is schema1 but has digest": {
+			ref:                "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			schema1:            true,
+			expectedRepoDigest: "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			expectedRepoTag:    "",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		named, err := docker.ParseDockerRef(test.ref)
+		assert.NoError(t, err)
+		repoDigest, repoTag := getRepoDigestAndTag(named, digest, test.schema1)
+		assert.Equal(t, test.expectedRepoDigest, repoDigest)
+		assert.Equal(t, test.expectedRepoTag, repoTag)
+	}
+}
+
+func TestBuildLabels(t *testing.T) {
+	configLabels := map[string]string{
+		"a": "b",
+		"c": "d",
+	}
+	newLabels := buildLabels(configLabels, containerKindSandbox)
+	assert.Len(t, newLabels, 3)
+	assert.Equal(t, "b", newLabels["a"])
+	assert.Equal(t, "d", newLabels["c"])
+	assert.Equal(t, containerKindSandbox, newLabels[containerKindLabel])
+
+	newLabels["a"] = "e"
+	assert.Empty(t, configLabels[containerKindLabel], "should not add new labels into original label")
+	assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
+}
+
+func TestParseImageReferences(t *testing.T) {
+	refs := []string{
+		"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"gcr.io/library/busybox:1.2",
+		"sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"arbitrary-ref",
+	}
+	expectedTags := []string{
+		"gcr.io/library/busybox:1.2",
+	}
+	expectedDigests := []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"}
+	tags, digests := parseImageReferences(refs)
+	assert.Equal(t, expectedTags, tags)
+	assert.Equal(t, expectedDigests, digests)
+}
+
+func TestLocalResolve(t *testing.T) {
+	image := imagestore.Image{
+		ID:      "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		ChainID: "test-chain-id-1",
+		References: []string{
+			"docker.io/library/busybox:latest",
+			"docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+		Size: 10,
+	}
+	c := newTestCRIService()
+	var err error
+	c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{image})
+	assert.NoError(t, err)
+
+	for _, ref := range []string{
+		"sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		"busybox",
+		"busybox:latest",
+		"busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"library/busybox",
+		"library/busybox:latest",
+		"library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"docker.io/busybox",
+		"docker.io/busybox:latest",
+		"docker.io/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"docker.io/library/busybox",
+		"docker.io/library/busybox:latest",
+		"docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+	} {
+		img, err := c.localResolve(ref)
+		assert.NoError(t, err)
+		assert.Equal(t, image, img)
+	}
+	img, err := c.localResolve("randomid")
+	assert.Equal(t, store.ErrNotExist, err)
+	assert.Equal(t, imagestore.Image{}, img)
+}
+
+func TestGenerateRuntimeOptions(t *testing.T) {
+	nilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+`
+	nonNilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.legacy.options]
+  Runtime = "legacy"
+  RuntimeRoot = "/legacy"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runc.options]
+  BinaryName = "runc"
+  Root = "/runc"
+  NoNewKeyring = true
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+[containerd.runtimes.runcv2.options]
+  BinaryName = "runc"
+  Root = "/runcv2"
+  NoNewKeyring = true
+`
+	var nilOptsConfig, nonNilOptsConfig criconfig.Config
+	_, err := toml.Decode(nilOpts, &nilOptsConfig)
+	require.NoError(t, err)
+	_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
+	require.NoError(t, err)
+	require.Len(t, nilOptsConfig.Runtimes, 3)
+	require.Len(t, nonNilOptsConfig.Runtimes, 3)
+
+	for desc, test := range map[string]struct {
+		r               criconfig.Runtime
+		c               criconfig.Config
+		expectedOptions interface{}
+	}{
+		"when options is nil, should return nil option for io.containerd.runc.v1": {
+			r:               nilOptsConfig.Runtimes["runc"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should return nil option for io.containerd.runc.v2": {
+			r:               nilOptsConfig.Runtimes["runcv2"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should use legacy fields for legacy runtime": {
+			r: nilOptsConfig.Runtimes["legacy"],
+			c: nilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				SystemdCgroup: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v1": {
+			r: nonNilOptsConfig.Runtimes["runc"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runc",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v2": {
+			r: nonNilOptsConfig.Runtimes["runcv2"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runcv2",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for legacy runtime": {
+			r: nonNilOptsConfig.Runtimes["legacy"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				Runtime:     "legacy",
+				RuntimeRoot: "/legacy",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			opts, err := generateRuntimeOptions(test.r, test.c)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedOptions, opts)
+		})
+	}
+}
+
+func TestEnvDeduplication(t *testing.T) {
+	for desc, test := range map[string]struct {
+		existing []string
+		kv       [][2]string
+		expected []string
+	}{
+		"single env": {
+			kv: [][2]string{
+				{"a", "b"},
+			},
+			expected: []string{"a=b"},
+		},
+		"multiple envs": {
+			kv: [][2]string{
+				{"a", "b"},
+				{"c", "d"},
+				{"e", "f"},
+			},
+			expected: []string{
+				"a=b",
+				"c=d",
+				"e=f",
+			},
+		},
+		"env override": {
+			kv: [][2]string{
+				{"k1", "v1"},
+				{"k2", "v2"},
+				{"k3", "v3"},
+				{"k3", "v4"},
+				{"k1", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v5",
+				"k2=v2",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+		"existing env": {
+			existing: []string{
+				"k1=v1",
+				"k2=v2",
+				"k3=v3",
+			},
+			kv: [][2]string{
+				{"k3", "v4"},
+				{"k2", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v1",
+				"k2=v5",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		var spec runtimespec.Spec
+		if len(test.existing) > 0 {
+			spec.Process = &runtimespec.Process{
+				Env: test.existing,
+			}
+		}
+		for _, kv := range test.kv {
+			oci.WithEnv([]string{kv[0] + "=" + kv[1]})(context.Background(), nil, nil, &spec)
+		}
+		assert.Equal(t, test.expected, spec.Process.Env)
+	}
+}
+
+func TestPassThroughAnnotationsFilter(t *testing.T) {
+	for desc, test := range map[string]struct {
+		podAnnotations         map[string]string
+		runtimePodAnnotations  []string
+		passthroughAnnotations map[string]string
+	}{
+		"should support direct match": {
+			podAnnotations:         map[string]string{"c": "d", "d": "e"},
+			runtimePodAnnotations:  []string{"c"},
+			passthroughAnnotations: map[string]string{"c": "d"},
+		},
+		"should support wildcard match": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*.f", "z*g", "y.c*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"y.ca": "b",
+			},
+		},
+		"should support wildcard match all": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+		},
+		"should support match including path separator": {
+			podAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchend.com/1end":   "3",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchmid.com/mid1":   "6",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchhead.com/head1": "9",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"1end/matchend":       "14",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"mid1/matchmid":       "17",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"head1/matchhead":     "20",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+				"nomatch/nomatch":     "24",
+			},
+			runtimePodAnnotations: []string{
+				"matchend.com/end*",
+				"matchmid.com/mi*d",
+				"matchhead.com/*head",
+				"matchall.com/*",
+				"end*/matchend",
+				"mi*d/matchmid",
+				"*head/matchhead",
+				"*/matchall",
+				"match*/match*",
+			},
+			passthroughAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			passthroughAnnotations := getPassthroughAnnotations(test.podAnnotations, test.runtimePodAnnotations)
+			assert.Equal(t, test.passthroughAnnotations, passthroughAnnotations)
+		})
+	}
+}
+
+func TestEnsureRemoveAllNotExist(t *testing.T) {
+	// should never return an error for a non-existent path
+	if err := ensureRemoveAll(context.Background(), "/non/existent/path"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithDir(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test-ensure-removeall-with-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ensureRemoveAll(context.Background(), dir); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithFile(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "test-ensure-removeall-with-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmp.Close()
+	if err := ensureRemoveAll(context.Background(), tmp.Name()); err != nil {
+		t.Fatal(err)
+	}
+}

pkg/server/helpers_windows.go

@@ -0,0 +1,170 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// openLogFile opens/creates a container log file.
+// It specifies `FILE_SHARE_DELETE` option to make sure
+// log files can be rotated by kubelet.
+// TODO(windows): Use golang support after 1.14. (https://github.com/golang/go/issues/32088)
+func openLogFile(path string) (*os.File, error) {
+	path = fixLongPath(path)
+	if len(path) == 0 {
+		return nil, syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return nil, err
+	}
+	createmode := uint32(syscall.OPEN_ALWAYS)
+	access := uint32(syscall.FILE_APPEND_DATA)
+	sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
+	h, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+	if err != nil {
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}
+
+// Copyright (c) 2009 The Go Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+//
+// This is copied from https://golang.org/src/path/filepath/path_windows.go.
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less then 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !filepath.IsAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}
+
+// ensureRemoveAll is a wrapper for os.RemoveAll on Windows.
+func ensureRemoveAll(_ context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/server/image_list.go

@@ -0,0 +1,38 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ListImages lists existing images.
+// TODO(random-liu): Add image list filters after CRI defines this more clear, and kubelet
+// actually needs it.
+func (c *criService) ListImages(ctx context.Context, r *runtime.ListImagesRequest) (*runtime.ListImagesResponse, error) {
+	imagesInStore := c.imageStore.List()
+
+	var images []*runtime.Image
+	for _, image := range imagesInStore {
+		// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
+		// doesn't exist?
+		images = append(images, toCRIImage(image))
+	}
+
+	return &runtime.ListImagesResponse{Images: images}, nil
+}

pkg/server/image_list_test.go

@@ -0,0 +1,113 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	imagestore "github.com/containerd/cri/pkg/store/image"
+)
+
+func TestListImages(t *testing.T) {
+	c := newTestCRIService()
+	imagesInStore := []imagestore.Image{
+		{
+			ID:      "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-1",
+			References: []string{
+				"gcr.io/library/busybox:latest",
+				"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 1000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "root",
+				},
+			},
+		},
+		{
+			ID:      "sha256:2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-2",
+			References: []string{
+				"gcr.io/library/alpine:latest",
+				"gcr.io/library/alpine@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 2000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "1234:1234",
+				},
+			},
+		},
+		{
+			ID:      "sha256:3123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-3",
+			References: []string{
+				"gcr.io/library/ubuntu:latest",
+				"gcr.io/library/ubuntu@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 3000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "nobody",
+				},
+			},
+		},
+	}
+	expect := []*runtime.Image{
+		{
+			Id:          "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/busybox:latest"},
+			RepoDigests: []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(1000),
+			Username:    "root",
+		},
+		{
+			Id:          "sha256:2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/alpine:latest"},
+			RepoDigests: []string{"gcr.io/library/alpine@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(2000),
+			Uid:         &runtime.Int64Value{Value: 1234},
+		},
+		{
+			Id:          "sha256:3123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/ubuntu:latest"},
+			RepoDigests: []string{"gcr.io/library/ubuntu@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(3000),
+			Username:    "nobody",
+		},
+	}
+
+	var err error
+	c.imageStore, err = imagestore.NewFakeStore(imagesInStore)
+	assert.NoError(t, err)
+
+	resp, err := c.ListImages(context.Background(), &runtime.ListImagesRequest{})
+	assert.NoError(t, err)
+	require.NotNil(t, resp)
+	images := resp.GetImages()
+	assert.Len(t, images, len(expect))
+	for _, i := range expect {
+		assert.Contains(t, images, i)
+	}
+}

pkg/server/image_pull.go

@@ -0,0 +1,519 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/log"
+	distribution "github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/imgcrypt"
+	"github.com/containerd/imgcrypt/images/encryption"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+)
+
+// For image management:
+// 1) We have an in-memory metadata index to:
+//   a. Maintain ImageID -> RepoTags, ImageID -> RepoDigset relationships; ImageID
+//   is the digest of image config, which conforms to oci image spec.
+//   b. Cache constant and useful information such as image chainID, config etc.
+//   c. An image will be added into the in-memory metadata only when it's successfully
+//   pulled and unpacked.
+//
+// 2) We use containerd image metadata store and content store:
+//   a. To resolve image reference (digest/tag) locally. During pulling image, we
+//   normalize the image reference provided by user, and put it into image metadata
+//   store with resolved descriptor. For the other operations, if image id is provided,
+//   we'll access the in-memory metadata index directly; if image reference is
+//   provided, we'll normalize it, resolve it in containerd image metadata store
+//   to get the image id.
+//   b. As the backup of in-memory metadata in 1). During startup, the in-memory
+//   metadata could be re-constructed from image metadata store + content store.
+//
+// Several problems with current approach:
+// 1) An entry in containerd image metadata store doesn't mean a "READY" (successfully
+// pulled and unpacked) image. E.g. during pulling, the client gets killed. In that case,
+// if we saw an image without snapshots or with in-complete contents during startup,
+// should we re-pull the image? Or should we remove the entry?
+//
+// yanxuean: We can't delete image directly, because we don't know if the image
+// is pulled by us. There are resource leakage.
+//
+// 2) Containerd suggests user to add entry before pulling the image. However if
+// an error occurs during the pulling, should we remove the entry from metadata
+// store? Or should we leave it there until next startup (resource leakage)?
+//
+// 3) The cri plugin only exposes "READY" (successfully pulled and unpacked) images
+// to the user, which are maintained in the in-memory metadata index. However, it's
+// still possible that someone else removes the content or snapshot by-pass the cri plugin,
+// how do we detect that and update the in-memory metadata correspondingly? Always
+// check whether corresponding snapshot is ready when reporting image status?
+//
+// 4) Is the content important if we cached necessary information in-memory
+// after we pull the image? How to manage the disk usage of contents? If some
+// contents are missing but snapshots are ready, is the image still "READY"?
+
+// PullImage pulls an image with authentication config.
+func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
+	imageRef := r.GetImage().GetImage()
+	namedRef, err := distribution.ParseDockerRef(imageRef)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse image reference %q", imageRef)
+	}
+	ref := namedRef.String()
+	if ref != imageRef {
+		log.G(ctx).Debugf("PullImage using normalized image ref: %q", ref)
+	}
+	var (
+		resolver = docker.NewResolver(docker.ResolverOptions{
+			Headers: c.config.Registry.Headers,
+			Hosts:   c.registryHosts(r.GetAuth()),
+		})
+		isSchema1    bool
+		imageHandler containerdimages.HandlerFunc = func(_ context.Context,
+			desc imagespec.Descriptor) ([]imagespec.Descriptor, error) {
+			if desc.MediaType == containerdimages.MediaTypeDockerSchema1Manifest {
+				isSchema1 = true
+			}
+			return nil, nil
+		}
+	)
+
+	pullOpts := []containerd.RemoteOpt{
+		containerd.WithSchema1Conversion,
+		containerd.WithResolver(resolver),
+		containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter),
+		containerd.WithPullUnpack,
+		containerd.WithPullLabel(imageLabelKey, imageLabelValue),
+		containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads),
+		containerd.WithImageHandler(imageHandler),
+	}
+
+	pullOpts = append(pullOpts, c.encryptedImagesPullOpts()...)
+	if !c.config.ContainerdConfig.DisableSnapshotAnnotations {
+		pullOpts = append(pullOpts,
+			containerd.WithImageHandlerWrapper(appendInfoHandlerWrapper(ref)))
+	}
+
+	if c.config.ContainerdConfig.DiscardUnpackedLayers {
+		// Allows GC to clean layers up from the content store after unpacking
+		pullOpts = append(pullOpts,
+			containerd.WithChildLabelMap(containerdimages.ChildGCLabelsFilterLayers))
+	}
+
+	image, err := c.client.Pull(ctx, ref, pullOpts...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to pull and unpack image %q", ref)
+	}
+
+	configDesc, err := image.Config(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "get image config descriptor")
+	}
+	imageID := configDesc.Digest.String()
+
+	repoDigest, repoTag := getRepoDigestAndTag(namedRef, image.Target().Digest, isSchema1)
+	for _, r := range []string{imageID, repoTag, repoDigest} {
+		if r == "" {
+			continue
+		}
+		if err := c.createImageReference(ctx, r, image.Target()); err != nil {
+			return nil, errors.Wrapf(err, "failed to create image reference %q", r)
+		}
+		// Update image store to reflect the newest state in containerd.
+		// No need to use `updateImage`, because the image reference must
+		// have been managed by the cri plugin.
+		if err := c.imageStore.Update(ctx, r); err != nil {
+			return nil, errors.Wrapf(err, "failed to update image store %q", r)
+		}
+	}
+
+	log.G(ctx).Debugf("Pulled image %q with image id %q, repo tag %q, repo digest %q", imageRef, imageID,
+		repoTag, repoDigest)
+	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
+	// in-memory image store, it's only for in-memory indexing. The image could be removed
+	// by someone else anytime, before/during/after we create the metadata. We should always
+	// check the actual state in containerd before using the image or returning status of the
+	// image.
+	return &runtime.PullImageResponse{ImageRef: imageID}, nil
+}
+
+// ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
+func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
+	if auth == nil {
+		return "", "", nil
+	}
+	if auth.ServerAddress != "" {
+		// Do not return the auth info when server address doesn't match.
+		u, err := url.Parse(auth.ServerAddress)
+		if err != nil {
+			return "", "", errors.Wrap(err, "parse server address")
+		}
+		if host != u.Host {
+			return "", "", nil
+		}
+	}
+	if auth.Username != "" {
+		return auth.Username, auth.Password, nil
+	}
+	if auth.IdentityToken != "" {
+		return "", auth.IdentityToken, nil
+	}
+	if auth.Auth != "" {
+		decLen := base64.StdEncoding.DecodedLen(len(auth.Auth))
+		decoded := make([]byte, decLen)
+		_, err := base64.StdEncoding.Decode(decoded, []byte(auth.Auth))
+		if err != nil {
+			return "", "", err
+		}
+		fields := strings.SplitN(string(decoded), ":", 2)
+		if len(fields) != 2 {
+			return "", "", errors.Errorf("invalid decoded auth: %q", decoded)
+		}
+		user, passwd := fields[0], fields[1]
+		return user, strings.Trim(passwd, "\x00"), nil
+	}
+	// TODO(random-liu): Support RegistryToken.
+	// An empty auth config is valid for anonymous registry
+	return "", "", nil
+}
+
+// createImageReference creates image reference inside containerd image store.
+// Note that because create and update are not finished in one transaction, there could be race. E.g.
+// the image reference is deleted by someone else after create returns already exists, but before update
+// happens.
+func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
+	img := containerdimages.Image{
+		Name:   name,
+		Target: desc,
+		// Add a label to indicate that the image is managed by the cri plugin.
+		Labels: map[string]string{imageLabelKey: imageLabelValue},
+	}
+	// TODO(random-liu): Figure out which is the more performant sequence create then update or
+	// update then create.
+	oldImg, err := c.client.ImageService().Create(ctx, img)
+	if err == nil || !errdefs.IsAlreadyExists(err) {
+		return err
+	}
+	if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[imageLabelKey] == imageLabelValue {
+		return nil
+	}
+	_, err = c.client.ImageService().Update(ctx, img, "target", "labels")
+	return err
+}
+
+// updateImage updates image store to reflect the newest state of an image reference
+// in containerd. If the reference is not managed by the cri plugin, the function also
+// generates necessary metadata for the image and make it managed.
+func (c *criService) updateImage(ctx context.Context, r string) error {
+	img, err := c.client.GetImage(ctx, r)
+	if err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrap(err, "get image by reference")
+	}
+	if err == nil && img.Labels()[imageLabelKey] != imageLabelValue {
+		// Make sure the image has the image id as its unique
+		// identifier that references the image in its lifetime.
+		configDesc, err := img.Config(ctx)
+		if err != nil {
+			return errors.Wrap(err, "get image id")
+		}
+		id := configDesc.Digest.String()
+		if err := c.createImageReference(ctx, id, img.Target()); err != nil {
+			return errors.Wrapf(err, "create image id reference %q", id)
+		}
+		if err := c.imageStore.Update(ctx, id); err != nil {
+			return errors.Wrapf(err, "update image store for %q", id)
+		}
+		// The image id is ready, add the label to mark the image as managed.
+		if err := c.createImageReference(ctx, r, img.Target()); err != nil {
+			return errors.Wrap(err, "create managed label")
+		}
+	}
+	// If the image is not found, we should continue updating the cache,
+	// so that the image can be removed from the cache.
+	if err := c.imageStore.Update(ctx, r); err != nil {
+		return errors.Wrapf(err, "update image store for %q", r)
+	}
+	return nil
+}
+
+// getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig
+func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.Config, error) {
+	var (
+		tlsConfig = &tls.Config{}
+		cert      tls.Certificate
+		err       error
+	)
+	if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile == "" {
+		return nil, errors.Errorf("cert file %q was specified, but no corresponding key file was specified", registryTLSConfig.CertFile)
+	}
+	if registryTLSConfig.CertFile == "" && registryTLSConfig.KeyFile != "" {
+		return nil, errors.Errorf("key file %q was specified, but no corresponding cert file was specified", registryTLSConfig.KeyFile)
+	}
+	if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile != "" {
+		cert, err = tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load cert file")
+		}
+		if len(cert.Certificate) != 0 {
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+		tlsConfig.BuildNameToCertificate() // nolint:staticcheck
+	}
+
+	if registryTLSConfig.CAFile != "" {
+		caCertPool, err := x509.SystemCertPool()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get system cert pool")
+		}
+		caCert, err := ioutil.ReadFile(registryTLSConfig.CAFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load CA file")
+		}
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	tlsConfig.InsecureSkipVerify = registryTLSConfig.InsecureSkipVerify
+	return tlsConfig, nil
+}
+
+// registryHosts is the registry hosts to be used by the resolver.
+func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts {
+	return func(host string) ([]docker.RegistryHost, error) {
+		var registries []docker.RegistryHost
+
+		endpoints, err := c.registryEndpoints(host)
+		if err != nil {
+			return nil, errors.Wrap(err, "get registry endpoints")
+		}
+		for _, e := range endpoints {
+			u, err := url.Parse(e)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e)
+			}
+
+			var (
+				transport = newTransport()
+				client    = &http.Client{Transport: transport}
+				config    = c.config.Registry.Configs[u.Host]
+			)
+
+			if config.TLS != nil {
+				transport.TLSClientConfig, err = c.getTLSConfig(*config.TLS)
+				if err != nil {
+					return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
+				}
+			}
+
+			if auth == nil && config.Auth != nil {
+				auth = toRuntimeAuthConfig(*config.Auth)
+			}
+
+			if u.Path == "" {
+				u.Path = "/v2"
+			}
+
+			registries = append(registries, docker.RegistryHost{
+				Client: client,
+				Authorizer: docker.NewDockerAuthorizer(
+					docker.WithAuthClient(client),
+					docker.WithAuthCreds(func(host string) (string, string, error) {
+						return ParseAuth(auth, host)
+					})),
+				Host:         u.Host,
+				Scheme:       u.Scheme,
+				Path:         u.Path,
+				Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull,
+			})
+		}
+		return registries, nil
+	}
+}
+
+// defaultScheme returns the default scheme for a registry host.
+func defaultScheme(host string) string {
+	if h, _, err := net.SplitHostPort(host); err == nil {
+		host = h
+	}
+	if host == "localhost" || host == "127.0.0.1" || host == "::1" {
+		return "http"
+	}
+	return "https"
+}
+
+// addDefaultScheme returns the endpoint with default scheme
+func addDefaultScheme(endpoint string) (string, error) {
+	if strings.Contains(endpoint, "://") {
+		return endpoint, nil
+	}
+	ue := "dummy://" + endpoint
+	u, err := url.Parse(ue)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s://%s", defaultScheme(u.Host), endpoint), nil
+}
+
+// registryEndpoints returns endpoints for a given host.
+// It adds default registry endpoint if it does not exist in the passed-in endpoint list.
+// It also supports wildcard host matching with `*`.
+func (c *criService) registryEndpoints(host string) ([]string, error) {
+	var endpoints []string
+	_, ok := c.config.Registry.Mirrors[host]
+	if ok {
+		endpoints = c.config.Registry.Mirrors[host].Endpoints
+	} else {
+		endpoints = c.config.Registry.Mirrors["*"].Endpoints
+	}
+	defaultHost, err := docker.DefaultHost(host)
+	if err != nil {
+		return nil, errors.Wrap(err, "get default host")
+	}
+	for i := range endpoints {
+		en, err := addDefaultScheme(endpoints[i])
+		if err != nil {
+			return nil, errors.Wrap(err, "parse endpoint url")
+		}
+		endpoints[i] = en
+	}
+	for _, e := range endpoints {
+		u, err := url.Parse(e)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse endpoint url")
+		}
+		if u.Host == host {
+			// Do not add default if the endpoint already exists.
+			return endpoints, nil
+		}
+	}
+	return append(endpoints, defaultScheme(defaultHost)+"://"+defaultHost), nil
+}
+
+// newTransport returns a new HTTP transport used to pull image.
+// TODO(random-liu): Create a library and share this code with `ctr`.
+func newTransport() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:       30 * time.Second,
+			KeepAlive:     30 * time.Second,
+			FallbackDelay: 300 * time.Millisecond,
+		}).DialContext,
+		MaxIdleConns:          10,
+		IdleConnTimeout:       30 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+	}
+}
+
+// encryptedImagesPullOpts returns the necessary list of pull options required
+// for decryption of encrypted images based on the cri decryption configuration.
+func (c *criService) encryptedImagesPullOpts() []containerd.RemoteOpt {
+	if c.config.ImageDecryption.KeyModel == criconfig.KeyModelNode {
+		ltdd := imgcrypt.Payload{}
+		decUnpackOpt := encryption.WithUnpackConfigApplyOpts(encryption.WithDecryptedUnpack(&ltdd))
+		opt := containerd.WithUnpackOpts([]containerd.UnpackOpt{decUnpackOpt})
+		return []containerd.RemoteOpt{opt}
+	}
+	return nil
+}
+
+const (
+	// targetRefLabel is a label which contains image reference and will be passed
+	// to snapshotters.
+	targetRefLabel = "containerd.io/snapshot/cri.image-ref"
+	// targetDigestLabel is a label which contains layer digest and will be passed
+	// to snapshotters.
+	targetDigestLabel = "containerd.io/snapshot/cri.layer-digest"
+	// targetImageLayersLabel is a label which contains layer digests contained in
+	// the target image and will be passed to snapshotters for preparing layers in
+	// parallel. Skipping some layers is allowed and only affects performance.
+	targetImageLayersLabel = "containerd.io/snapshot/cri.image-layers"
+)
+
+// appendInfoHandlerWrapper makes a handler which appends some basic information
+// of images to each layer descriptor as annotations during unpack. These
+// annotations will be passed to snapshotters as labels. These labels will be
+// used mainly by stargz-based snapshotters for querying image contents from the
+// registry.
+func appendInfoHandlerWrapper(ref string) func(f containerdimages.Handler) containerdimages.Handler {
+	return func(f containerdimages.Handler) containerdimages.Handler {
+		return containerdimages.HandlerFunc(func(ctx context.Context, desc imagespec.Descriptor) ([]imagespec.Descriptor, error) {
+			children, err := f.Handle(ctx, desc)
+			if err != nil {
+				return nil, err
+			}
+			switch desc.MediaType {
+			case imagespec.MediaTypeImageManifest, containerdimages.MediaTypeDockerSchema2Manifest:
+				for i := range children {
+					c := &children[i]
+					if containerdimages.IsLayerType(c.MediaType) {
+						if c.Annotations == nil {
+							c.Annotations = make(map[string]string)
+						}
+						c.Annotations[targetRefLabel] = ref
+						c.Annotations[targetDigestLabel] = c.Digest.String()
+						c.Annotations[targetImageLayersLabel] = getLayers(ctx, targetImageLayersLabel, children[i:], labels.Validate)
+					}
+				}
+			}
+			return children, nil
+		})
+	}
+}
+
+// getLayers returns comma-separated digests based on the passed list of
+// descriptors. The returned list contains as many digests as possible as well
+// as meets the label validation.
+func getLayers(ctx context.Context, key string, descs []imagespec.Descriptor, validate func(k, v string) error) (layers string) {
+	var item string
+	for _, l := range descs {
+		if containerdimages.IsLayerType(l.MediaType) {
+			item = l.Digest.String()
+			if layers != "" {
+				item = "," + item
+			}
+			// This avoids the label hits the size limitation.
+			if err := validate(key, layers+item); err != nil {
+				log.G(ctx).WithError(err).WithField("label", key).Debugf("%q is omitted in the layers list", l.Digest.String())
+				break
+			}
+			layers += item
+		}
+	}
+	return
+}

pkg/server/image_pull_test.go

@@ -0,0 +1,379 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"testing"
+
+	digest "github.com/opencontainers/go-digest"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/cri/pkg/config"
+)
+
+func TestParseAuth(t *testing.T) {
+	testUser := "username"
+	testPasswd := "password"
+	testAuthLen := base64.StdEncoding.EncodedLen(len(testUser + ":" + testPasswd))
+	testAuth := make([]byte, testAuthLen)
+	base64.StdEncoding.Encode(testAuth, []byte(testUser+":"+testPasswd))
+	invalidAuth := make([]byte, testAuthLen)
+	base64.StdEncoding.Encode(invalidAuth, []byte(testUser+"@"+testPasswd))
+	for desc, test := range map[string]struct {
+		auth           *runtime.AuthConfig
+		host           string
+		expectedUser   string
+		expectedSecret string
+		expectErr      bool
+	}{
+		"should not return error if auth config is nil": {},
+		"should not return error if empty auth is provided for access to anonymous registry": {
+			auth:      &runtime.AuthConfig{},
+			expectErr: false,
+		},
+		"should support identity token": {
+			auth:           &runtime.AuthConfig{IdentityToken: "abcd"},
+			expectedSecret: "abcd",
+		},
+		"should support username and password": {
+			auth: &runtime.AuthConfig{
+				Username: testUser,
+				Password: testPasswd,
+			},
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should support auth": {
+			auth:           &runtime.AuthConfig{Auth: string(testAuth)},
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should return error for invalid auth": {
+			auth:      &runtime.AuthConfig{Auth: string(invalidAuth)},
+			expectErr: true,
+		},
+		"should return empty auth if server address doesn't match": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-2.io",
+			expectedUser:   "",
+			expectedSecret: "",
+		},
+		"should return auth if server address matches": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should return auth if server address is not specified": {
+			auth: &runtime.AuthConfig{
+				Username: testUser,
+				Password: testPasswd,
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		u, s, err := ParseAuth(test.auth, test.host)
+		assert.Equal(t, test.expectErr, err != nil)
+		assert.Equal(t, test.expectedUser, u)
+		assert.Equal(t, test.expectedSecret, s)
+	}
+}
+
+func TestRegistryEndpoints(t *testing.T) {
+	for desc, test := range map[string]struct {
+		mirrors  map[string]criconfig.Mirror
+		host     string
+		expected []string
+	}{
+		"no mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-1.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-3.io",
+			},
+		},
+		"mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"wildcard mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"*": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"host should take precedence if both host and wildcard mirrors are configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"*": {
+					Endpoints: []string{
+						"https://registry-1.io",
+					},
+				},
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"default endpoint in list with http": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"http://registry-3.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"http://registry-3.io",
+			},
+		},
+		"default endpoint in list with https": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"https://registry-3.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"default endpoint in list with path": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"https://registry-3.io/path",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io/path",
+			},
+		},
+		"miss scheme endpoint in list with path": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-3.io",
+						"registry-1.io",
+						"127.0.0.1:1234",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-3.io",
+				"https://registry-1.io",
+				"http://127.0.0.1:1234",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		c.config.Registry.Mirrors = test.mirrors
+		got, err := c.registryEndpoints(test.host)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestDefaultScheme(t *testing.T) {
+	for desc, test := range map[string]struct {
+		host     string
+		expected string
+	}{
+		"should use http by default for localhost": {
+			host:     "localhost",
+			expected: "http",
+		},
+		"should use http by default for localhost with port": {
+			host:     "localhost:8080",
+			expected: "http",
+		},
+		"should use http by default for 127.0.0.1": {
+			host:     "127.0.0.1",
+			expected: "http",
+		},
+		"should use http by default for 127.0.0.1 with port": {
+			host:     "127.0.0.1:8080",
+			expected: "http",
+		},
+		"should use http by default for ::1": {
+			host:     "::1",
+			expected: "http",
+		},
+		"should use http by default for ::1 with port": {
+			host:     "[::1]:8080",
+			expected: "http",
+		},
+		"should use https by default for remote host": {
+			host:     "remote",
+			expected: "https",
+		},
+		"should use https by default for remote host with port": {
+			host:     "remote:8080",
+			expected: "https",
+		},
+		"should use https by default for remote ip": {
+			host:     "8.8.8.8",
+			expected: "https",
+		},
+		"should use https by default for remote ip with port": {
+			host:     "8.8.8.8:8080",
+			expected: "https",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got := defaultScheme(test.host)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestEncryptedImagePullOpts(t *testing.T) {
+	for desc, test := range map[string]struct {
+		keyModel     string
+		expectedOpts int
+	}{
+		"node key model should return one unpack opt": {
+			keyModel:     criconfig.KeyModelNode,
+			expectedOpts: 1,
+		},
+		"no key model selected should default to node key model": {
+			keyModel:     "",
+			expectedOpts: 0,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		c.config.ImageDecryption.KeyModel = test.keyModel
+		got := len(c.encryptedImagesPullOpts())
+		assert.Equal(t, test.expectedOpts, got)
+	}
+}
+
+func TestImageLayersLabel(t *testing.T) {
+	sampleKey := "sampleKey"
+	sampleDigest, err := digest.Parse("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	assert.NoError(t, err)
+	sampleMaxSize := 300
+	sampleValidate := func(k, v string) error {
+		if (len(k) + len(v)) > sampleMaxSize {
+			return fmt.Errorf("invalid: %q: %q", k, v)
+		}
+		return nil
+	}
+
+	tests := []struct {
+		name      string
+		layersNum int
+		wantNum   int
+	}{
+		{
+			name:      "valid number of layers",
+			layersNum: 2,
+			wantNum:   2,
+		},
+		{
+			name:      "many layers",
+			layersNum: 5, // hits sampleMaxSize (300 chars).
+			wantNum:   4, // layers should be omitted for avoiding invalid label.
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var sampleLayers []imagespec.Descriptor
+			for i := 0; i < tt.layersNum; i++ {
+				sampleLayers = append(sampleLayers, imagespec.Descriptor{
+					MediaType: imagespec.MediaTypeImageLayerGzip,
+					Digest:    sampleDigest,
+				})
+			}
+			gotS := getLayers(context.Background(), sampleKey, sampleLayers, sampleValidate)
+			got := len(strings.Split(gotS, ","))
+			assert.Equal(t, tt.wantNum, got)
+		})
+	}
+}

pkg/server/image_remove.go

@@ -0,0 +1,65 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/store"
+)
+
+// RemoveImage removes the image.
+// TODO(random-liu): Update CRI to pass image reference instead of ImageSpec. (See
+// kubernetes/kubernetes#46255)
+// TODO(random-liu): We should change CRI to distinguish image id and image spec.
+// Remove the whole image no matter the it's image id or reference. This is the
+// semantic defined in CRI now.
+func (c *criService) RemoveImage(ctx context.Context, r *runtime.RemoveImageRequest) (*runtime.RemoveImageResponse, error) {
+	image, err := c.localResolve(r.GetImage().GetImage())
+	if err != nil {
+		if err == store.ErrNotExist {
+			// return empty without error when image not found.
+			return &runtime.RemoveImageResponse{}, nil
+		}
+		return nil, errors.Wrapf(err, "can not resolve %q locally", r.GetImage().GetImage())
+	}
+
+	// Remove all image references.
+	for i, ref := range image.References {
+		var opts []images.DeleteOpt
+		if i == len(image.References)-1 {
+			// Delete the last image reference synchronously to trigger garbage collection.
+			// This is best effort. It is possible that the image reference is deleted by
+			// someone else before this point.
+			opts = []images.DeleteOpt{images.SynchronousDelete()}
+		}
+		err = c.client.ImageService().Delete(ctx, ref, opts...)
+		if err == nil || errdefs.IsNotFound(err) {
+			// Update image store to reflect the newest state in containerd.
+			if err := c.imageStore.Update(ctx, ref); err != nil {
+				return nil, errors.Wrapf(err, "failed to update image reference %q for %q", ref, image.ID)
+			}
+			continue
+		}
+		return nil, errors.Wrapf(err, "failed to delete image reference %q for %q", ref, image.ID)
+	}
+	return &runtime.RemoveImageResponse{}, nil
+}

pkg/server/image_status.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/containerd/containerd/log"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/cri/pkg/store"
+	imagestore "github.com/containerd/cri/pkg/store/image"
+)
+
+// ImageStatus returns the status of the image, returns nil if the image isn't present.
+// TODO(random-liu): We should change CRI to distinguish image id and image spec. (See
+// kubernetes/kubernetes#46255)
+func (c *criService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (*runtime.ImageStatusResponse, error) {
+	image, err := c.localResolve(r.GetImage().GetImage())
+	if err != nil {
+		if err == store.ErrNotExist {
+			// return empty without error when image not found.
+			return &runtime.ImageStatusResponse{}, nil
+		}
+		return nil, errors.Wrapf(err, "can not resolve %q locally", r.GetImage().GetImage())
+	}
+	// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
+	// doesn't exist?
+
+	runtimeImage := toCRIImage(image)
+	info, err := c.toCRIImageInfo(ctx, &image, r.GetVerbose())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate image info")
+	}
+
+	return &runtime.ImageStatusResponse{
+		Image: runtimeImage,
+		Info:  info,
+	}, nil
+}
+
+// toCRIImage converts internal image object to CRI runtime.Image.
+func toCRIImage(image imagestore.Image) *runtime.Image {
+	repoTags, repoDigests := parseImageReferences(image.References)
+	runtimeImage := &runtime.Image{
+		Id:          image.ID,
+		RepoTags:    repoTags,
+		RepoDigests: repoDigests,
+		Size_:       uint64(image.Size),
+	}
+	uid, username := getUserFromImage(image.ImageSpec.Config.User)
+	if uid != nil {
+		runtimeImage.Uid = &runtime.Int64Value{Value: *uid}
+	}
+	runtimeImage.Username = username
+
+	return runtimeImage
+}
+
+// TODO (mikebrow): discuss moving this struct and / or constants for info map for some or all of these fields to CRI
+type verboseImageInfo struct {
+	ChainID   string          `json:"chainID"`
+	ImageSpec imagespec.Image `json:"imageSpec"`
+}
+
+// toCRIImageInfo converts internal image object information to CRI image status response info map.
+func (c *criService) toCRIImageInfo(ctx context.Context, image *imagestore.Image, verbose bool) (map[string]string, error) {
+	if !verbose {
+		return nil, nil
+	}
+
+	info := make(map[string]string)
+
+	imi := &verboseImageInfo{
+		ChainID:   image.ChainID,
+		ImageSpec: image.ImageSpec,
+	}
+
+	m, err := json.Marshal(imi)
+	if err == nil {
+		info["info"] = string(m)
+	} else {
+		log.G(ctx).WithError(err).Errorf("failed to marshal info %v", imi)
+		info["info"] = err.Error()
+	}
+
+	return info, nil
+}

pkg/server/image_status_test.go

@@ -0,0 +1,74 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	imagestore "github.com/containerd/cri/pkg/store/image"
+)
+
+func TestImageStatus(t *testing.T) {
+	testID := "sha256:d848ce12891bf78792cda4a23c58984033b0c397a55e93a1556202222ecc5ed4"
+	image := imagestore.Image{
+		ID:      testID,
+		ChainID: "test-chain-id",
+		References: []string{
+			"gcr.io/library/busybox:latest",
+			"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+		Size: 1234,
+		ImageSpec: imagespec.Image{
+			Config: imagespec.ImageConfig{
+				User: "user:group",
+			},
+		},
+	}
+	expected := &runtime.Image{
+		Id:          testID,
+		RepoTags:    []string{"gcr.io/library/busybox:latest"},
+		RepoDigests: []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+		Size_:       uint64(1234),
+		Username:    "user",
+	}
+
+	c := newTestCRIService()
+	t.Logf("should return nil image spec without error for non-exist image")
+	resp, err := c.ImageStatus(context.Background(), &runtime.ImageStatusRequest{
+		Image: &runtime.ImageSpec{Image: testID},
+	})
+	assert.NoError(t, err)
+	require.NotNil(t, resp)
+	assert.Nil(t, resp.GetImage())
+
+	c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{image})
+	assert.NoError(t, err)
+
+	t.Logf("should return correct image status for exist image")
+	resp, err = c.ImageStatus(context.Background(), &runtime.ImageStatusRequest{
+		Image: &runtime.ImageSpec{Image: testID},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, resp)
+	assert.Equal(t, expected, resp.GetImage())
+}

pkg/server/imagefs_info.go

@@ -0,0 +1,52 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"time"
+
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+// TODO(windows): Usage for windows is always 0 right now. Support this for windows.
+func (c *criService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (*runtime.ImageFsInfoResponse, error) {
+	snapshots := c.snapshotStore.List()
+	timestamp := time.Now().UnixNano()
+	var usedBytes, inodesUsed uint64
+	for _, sn := range snapshots {
+		// Use the oldest timestamp as the timestamp of imagefs info.
+		if sn.Timestamp < timestamp {
+			timestamp = sn.Timestamp
+		}
+		usedBytes += sn.Size
+		inodesUsed += sn.Inodes
+	}
+	// TODO(random-liu): Handle content store
+	return &runtime.ImageFsInfoResponse{
+		ImageFilesystems: []*runtime.FilesystemUsage{
+			{
+				Timestamp:  timestamp,
+				FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPath},
+				UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+				InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+			},
+		},
+	}, nil
+}