Move cri server packages under pkg/cri

Organizes the cri related server packages under pkg/cri

Signed-off-by: Derek McGowan <derek@mcg.dev>

pkg/cri/server/bandwidth/doc.go

@@ -0,0 +1,34 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bandwidth provides utilities for bandwidth shaping
+package bandwidth

pkg/cri/server/bandwidth/fake_shaper.go

@@ -0,0 +1,72 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"errors"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// FakeShaper provides an implementation of the bandwith.Shaper.
+// Beware this is implementation has no features besides Reset and GetCIDRs.
+type FakeShaper struct {
+	CIDRs      []string
+	ResetCIDRs []string
+}
+
+// Limit is not implemented
+func (f *FakeShaper) Limit(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+// Reset appends a particular CIDR to the set of ResetCIDRs being managed by this shaper
+func (f *FakeShaper) Reset(cidr string) error {
+	f.ResetCIDRs = append(f.ResetCIDRs, cidr)
+	return nil
+}
+
+// ReconcileInterface is not implemented
+func (f *FakeShaper) ReconcileInterface() error {
+	return errors.New("unimplemented")
+}
+
+// ReconcileCIDR is not implemented
+func (f *FakeShaper) ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+// GetCIDRs returns the set of CIDRs that are being managed by this shaper
+func (f *FakeShaper) GetCIDRs() ([]string, error) {
+	return f.CIDRs, nil
+}

pkg/cri/server/bandwidth/interfaces.go

@@ -0,0 +1,56 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import "k8s.io/apimachinery/pkg/api/resource"
+
+// Shaper is designed so that the shaper structs created
+// satisfy the Shaper interface.
+type Shaper interface {
+	// Limit the bandwidth for a particular CIDR on a particular interface
+	//   * ingress and egress are in bits/second
+	//   * cidr is expected to be a valid network CIDR (e.g. '1.2.3.4/32' or '10.20.0.1/16')
+	// 'egress' bandwidth limit applies to all packets on the interface whose source matches 'cidr'
+	// 'ingress' bandwidth limit applies to all packets on the interface whose destination matches 'cidr'
+	// Limits are aggregate limits for the CIDR, not per IP address.  CIDRs must be unique, but can be overlapping, traffic
+	// that matches multiple CIDRs counts against all limits.
+	Limit(cidr string, egress, ingress *resource.Quantity) error
+	// Remove a bandwidth limit for a particular CIDR on a particular network interface
+	Reset(cidr string) error
+	// Reconcile the interface managed by this shaper with the state on the ground.
+	ReconcileInterface() error
+	// Reconcile a CIDR managed by this shaper with the state on the ground
+	ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error
+	// GetCIDRs returns the set of CIDRs that are being managed by this shaper
+	GetCIDRs() ([]string, error)
+}

pkg/cri/server/bandwidth/linux.go

@@ -0,0 +1,361 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/utils/exec"
+
+	"k8s.io/klog/v2"
+)
+
+var (
+	classShowMatcher      = regexp.MustCompile(`class htb (1:\d+)`)
+	classAndHandleMatcher = regexp.MustCompile(`filter parent 1:.*fh (\d+::\d+).*flowid (\d+:\d+)`)
+)
+
+// tcShaper provides an implementation of the Shaper interface on Linux using the 'tc' tool.
+// In general, using this requires that the caller posses the NET_CAP_ADMIN capability, though if you
+// do this within an container, it only requires the NS_CAPABLE capability for manipulations to that
+// container's network namespace.
+// Uses the hierarchical token bucket queuing discipline (htb), this requires Linux 2.4.20 or newer
+// or a custom kernel with that queuing discipline backported.
+type tcShaper struct {
+	e     exec.Interface
+	iface string
+}
+
+// NewTCShaper makes a new tcShaper for the given interface
+func NewTCShaper(iface string) Shaper {
+	shaper := &tcShaper{
+		e:     exec.New(),
+		iface: iface,
+	}
+	return shaper
+}
+
+func (t *tcShaper) execAndLog(cmdStr string, args ...string) error {
+	klog.V(6).Infof("Running: %s %s", cmdStr, strings.Join(args, " "))
+	cmd := t.e.Command(cmdStr, args...)
+	out, err := cmd.CombinedOutput()
+	klog.V(6).Infof("Output from tc: %s", string(out))
+	return err
+}
+
+func (t *tcShaper) nextClassID() (int, error) {
+	data, err := t.e.Command("tc", "class", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return -1, err
+	}
+
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	classes := sets.String{}
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		// skip empty lines
+		if len(line) == 0 {
+			continue
+		}
+		// expected tc line:
+		// class htb 1:1 root prio 0 rate 1000Kbit ceil 1000Kbit burst 1600b cburst 1600b
+		matches := classShowMatcher.FindStringSubmatch(line)
+		if len(matches) != 2 {
+			return -1, fmt.Errorf("unexpected output from tc: %s (%v)", scanner.Text(), matches)
+		}
+		classes.Insert(matches[1])
+	}
+
+	// Make sure it doesn't go forever
+	for nextClass := 1; nextClass < 10000; nextClass++ {
+		if !classes.Has(fmt.Sprintf("1:%d", nextClass)) {
+			return nextClass, nil
+		}
+	}
+	// This should really never happen
+	return -1, fmt.Errorf("exhausted class space, please try again")
+}
+
+// Convert a CIDR from text to a hex representation
+// Strips any masked parts of the IP, so 1.2.3.4/16 becomes hex(1.2.0.0)/ffffffff
+func hexCIDR(cidr string) (string, error) {
+	ip, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return "", err
+	}
+	ip = ip.Mask(ipnet.Mask)
+	hexIP := hex.EncodeToString([]byte(ip))
+	hexMask := ipnet.Mask.String()
+	return hexIP + "/" + hexMask, nil
+}
+
+// Convert a CIDR from hex representation to text, opposite of the above.
+func asciiCIDR(cidr string) (string, error) {
+	parts := strings.Split(cidr, "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected CIDR format: %s", cidr)
+	}
+	ipData, err := hex.DecodeString(parts[0])
+	if err != nil {
+		return "", err
+	}
+	ip := net.IP(ipData)
+
+	maskData, err := hex.DecodeString(parts[1])
+	if err != nil {
+		return "", err
+	}
+	mask := net.IPMask(maskData)
+	size, _ := mask.Size()
+
+	return fmt.Sprintf("%s/%d", ip.String(), size), nil
+}
+
+func (t *tcShaper) findCIDRClass(cidr string) (classAndHandleList [][]string, found bool, err error) {
+	data, err := t.e.Command("tc", "filter", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return classAndHandleList, false, err
+	}
+
+	hex, err := hexCIDR(cidr)
+	if err != nil {
+		return classAndHandleList, false, err
+	}
+	spec := fmt.Sprintf("match %s", hex)
+
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	filter := ""
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		if strings.HasPrefix(line, "filter") {
+			filter = line
+			continue
+		}
+		if strings.Contains(line, spec) {
+			// expected tc line:
+			// `filter parent 1: protocol ip pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1` (old version) or
+			// `filter parent 1: protocol ip pref 1 u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 not_in_hw` (new version)
+			matches := classAndHandleMatcher.FindStringSubmatch(filter)
+			if len(matches) != 3 {
+				return classAndHandleList, false, fmt.Errorf("unexpected output from tc: %s %d (%v)", filter, len(matches), matches)
+			}
+			resultTmp := []string{matches[2], matches[1]}
+			classAndHandleList = append(classAndHandleList, resultTmp)
+		}
+	}
+	if len(classAndHandleList) > 0 {
+		return classAndHandleList, true, nil
+	}
+	return classAndHandleList, false, nil
+}
+
+func makeKBitString(rsrc *resource.Quantity) string {
+	return fmt.Sprintf("%dkbit", (rsrc.Value() / 1000))
+}
+
+func (t *tcShaper) makeNewClass(rate string) (int, error) {
+	class, err := t.nextClassID()
+	if err != nil {
+		return -1, err
+	}
+	if err := t.execAndLog("tc", "class", "add",
+		"dev", t.iface,
+		"parent", "1:",
+		"classid", fmt.Sprintf("1:%d", class),
+		"htb", "rate", rate); err != nil {
+		return -1, err
+	}
+	return class, nil
+}
+
+func (t *tcShaper) Limit(cidr string, upload, download *resource.Quantity) (err error) {
+	var downloadClass, uploadClass int
+	if download != nil {
+		if downloadClass, err = t.makeNewClass(makeKBitString(download)); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "filter", "add",
+			"dev", t.iface,
+			"protocol", "ip",
+			"parent", "1:0",
+			"prio", "1", "u32",
+			"match", "ip", "dst", cidr,
+			"flowid", fmt.Sprintf("1:%d", downloadClass)); err != nil {
+			return err
+		}
+	}
+	if upload != nil {
+		if uploadClass, err = t.makeNewClass(makeKBitString(upload)); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "filter", "add",
+			"dev", t.iface,
+			"protocol", "ip",
+			"parent", "1:0",
+			"prio", "1", "u32",
+			"match", "ip", "src", cidr,
+			"flowid", fmt.Sprintf("1:%d", uploadClass)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// tests to see if an interface exists, if it does, return true and the status line for the interface
+// returns false, "", <err> if an error occurs.
+func (t *tcShaper) interfaceExists() (bool, string, error) {
+	data, err := t.e.Command("tc", "qdisc", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return false, "", err
+	}
+	value := strings.TrimSpace(string(data))
+	if len(value) == 0 {
+		return false, "", nil
+	}
+	// Newer versions of tc and/or the kernel return the following instead of nothing:
+	// qdisc noqueue 0: root refcnt 2
+	fields := strings.Fields(value)
+	if len(fields) > 1 && fields[1] == "noqueue" {
+		return false, "", nil
+	}
+	return true, value, nil
+}
+
+func (t *tcShaper) ReconcileCIDR(cidr string, upload, download *resource.Quantity) error {
+	_, found, err := t.findCIDRClass(cidr)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return t.Limit(cidr, upload, download)
+	}
+	// TODO: actually check bandwidth limits here
+	return nil
+}
+
+func (t *tcShaper) ReconcileInterface() error {
+	exists, output, err := t.interfaceExists()
+	if err != nil {
+		return err
+	}
+	if !exists {
+		klog.V(4).Info("Didn't find bandwidth interface, creating")
+		return t.initializeInterface()
+	}
+	fields := strings.Split(output, " ")
+	if len(fields) < 12 || fields[1] != "htb" || fields[2] != "1:" {
+		if err := t.deleteInterface(fields[2]); err != nil {
+			return err
+		}
+		return t.initializeInterface()
+	}
+	return nil
+}
+
+func (t *tcShaper) initializeInterface() error {
+	return t.execAndLog("tc", "qdisc", "add", "dev", t.iface, "root", "handle", "1:", "htb", "default", "30")
+}
+
+func (t *tcShaper) Reset(cidr string) error {
+	classAndHandle, found, err := t.findCIDRClass(cidr)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("Failed to find cidr: %s on interface: %s", cidr, t.iface)
+	}
+	for i := 0; i < len(classAndHandle); i++ {
+		if err := t.execAndLog("tc", "filter", "del",
+			"dev", t.iface,
+			"parent", "1:",
+			"proto", "ip",
+			"prio", "1",
+			"handle", classAndHandle[i][1], "u32"); err != nil {
+			return err
+		}
+		if err := t.execAndLog("tc", "class", "del",
+			"dev", t.iface,
+			"parent", "1:",
+			"classid", classAndHandle[i][0]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *tcShaper) deleteInterface(class string) error {
+	return t.execAndLog("tc", "qdisc", "delete", "dev", t.iface, "root", "handle", class)
+}
+
+func (t *tcShaper) GetCIDRs() ([]string, error) {
+	data, err := t.e.Command("tc", "filter", "show", "dev", t.iface).CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+
+	result := []string{}
+	scanner := bufio.NewScanner(bytes.NewBuffer(data))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		if strings.Contains(line, "match") {
+			parts := strings.Split(line, " ")
+			// expected tc line:
+			// match <cidr> at <number>
+			if len(parts) != 4 {
+				return nil, fmt.Errorf("unexpected output: %v", parts)
+			}
+			cidr, err := asciiCIDR(parts[1])
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, cidr)
+		}
+	}
+	return result, nil
+}

pkg/cri/server/bandwidth/unsupported.go

@@ -0,0 +1,69 @@
+// +build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"errors"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+type unsupportedShaper struct {
+}
+
+// NewTCShaper makes a new unsupportedShapper for the given interface
+func NewTCShaper(iface string) Shaper {
+	return &unsupportedShaper{}
+}
+
+func (f *unsupportedShaper) Limit(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) Reset(cidr string) error {
+	return nil
+}
+
+func (f *unsupportedShaper) ReconcileInterface() error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) ReconcileCIDR(cidr string, egress, ingress *resource.Quantity) error {
+	return errors.New("unimplemented")
+}
+
+func (f *unsupportedShaper) GetCIDRs() ([]string, error) {
+	return []string{}, nil
+}

pkg/cri/server/bandwidth/utils.go

@@ -0,0 +1,82 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bandwidth
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var minRsrc = resource.MustParse("1k")
+var maxRsrc = resource.MustParse("1P")
+
+func validateBandwidthIsReasonable(rsrc *resource.Quantity) error {
+	if rsrc.Value() < minRsrc.Value() {
+		return fmt.Errorf("resource is unreasonably small (< 1kbit)")
+	}
+	if rsrc.Value() > maxRsrc.Value() {
+		return fmt.Errorf("resoruce is unreasonably large (> 1Pbit)")
+	}
+	return nil
+}
+
+// ExtractPodBandwidthResources extracts the ingress and egress from the given pod annotations
+func ExtractPodBandwidthResources(podAnnotations map[string]string) (ingress, egress *resource.Quantity, err error) {
+	if podAnnotations == nil {
+		return nil, nil, nil
+	}
+	str, found := podAnnotations["kubernetes.io/ingress-bandwidth"]
+	if found {
+		ingressValue, err := resource.ParseQuantity(str)
+		if err != nil {
+			return nil, nil, err
+		}
+		ingress = &ingressValue
+		if err := validateBandwidthIsReasonable(ingress); err != nil {
+			return nil, nil, err
+		}
+	}
+	str, found = podAnnotations["kubernetes.io/egress-bandwidth"]
+	if found {
+		egressValue, err := resource.ParseQuantity(str)
+		if err != nil {
+			return nil, nil, err
+		}
+		egress = &egressValue
+		if err := validateBandwidthIsReasonable(egress); err != nil {
+			return nil, nil, err
+		}
+	}
+	return ingress, egress, nil
+}

pkg/cri/server/cni_conf_syncer.go

@@ -0,0 +1,121 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"sync"
+
+	cni "github.com/containerd/go-cni"
+	"github.com/fsnotify/fsnotify"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// cniNetConfSyncer is used to reload cni network conf triggered by fs change
+// events.
+type cniNetConfSyncer struct {
+	// only used for lastSyncStatus
+	sync.RWMutex
+	lastSyncStatus error
+
+	watcher   *fsnotify.Watcher
+	confDir   string
+	netPlugin cni.CNI
+	loadOpts  []cni.CNIOpt
+}
+
+// newCNINetConfSyncer creates cni network conf syncer.
+func newCNINetConfSyncer(confDir string, netPlugin cni.CNI, loadOpts []cni.CNIOpt) (*cniNetConfSyncer, error) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create fsnotify watcher")
+	}
+
+	if err := os.MkdirAll(confDir, 0700); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni conf dir=%s for watch", confDir)
+	}
+
+	if err := watcher.Add(confDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to watch cni conf dir %s", confDir)
+	}
+
+	syncer := &cniNetConfSyncer{
+		watcher:   watcher,
+		confDir:   confDir,
+		netPlugin: netPlugin,
+		loadOpts:  loadOpts,
+	}
+
+	if err := syncer.netPlugin.Load(syncer.loadOpts...); err != nil {
+		logrus.WithError(err).Error("failed to load cni during init, please check CRI plugin status before setting up network for pods")
+		syncer.updateLastStatus(err)
+	}
+	return syncer, nil
+}
+
+// syncLoop monitors any fs change events from cni conf dir and tries to reload
+// cni configuration.
+func (syncer *cniNetConfSyncer) syncLoop() error {
+	for {
+		select {
+		case event := <-syncer.watcher.Events:
+			// Only reload config when receiving write/rename/remove
+			// events
+			//
+			// TODO(fuweid): Might only reload target cni config
+			// files to prevent no-ops.
+			if event.Op&(fsnotify.Chmod|fsnotify.Create) > 0 {
+				logrus.Debugf("ignore event from cni conf dir: %s", event)
+				continue
+			}
+			logrus.Debugf("receiving change event from cni conf dir: %s", event)
+
+			lerr := syncer.netPlugin.Load(syncer.loadOpts...)
+			if lerr != nil {
+				logrus.WithError(lerr).
+					Errorf("failed to reload cni configuration after receiving fs change event(%s)", event)
+			}
+			syncer.updateLastStatus(lerr)
+
+		case err := <-syncer.watcher.Errors:
+			if err != nil {
+				logrus.WithError(err).Error("failed to continue sync cni conf change")
+				return err
+			}
+		}
+	}
+}
+
+// lastStatus retrieves last sync status.
+func (syncer *cniNetConfSyncer) lastStatus() error {
+	syncer.RLock()
+	defer syncer.RUnlock()
+	return syncer.lastSyncStatus
+}
+
+// updateLastStatus will be called after every single cni load.
+func (syncer *cniNetConfSyncer) updateLastStatus(err error) {
+	syncer.Lock()
+	defer syncer.Unlock()
+	syncer.lastSyncStatus = err
+}
+
+// stop stops watcher in the syncLoop.
+func (syncer *cniNetConfSyncer) stop() error {
+	return syncer.watcher.Close()
+}

pkg/cri/server/container_attach.go

@@ -0,0 +1,84 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"k8s.io/client-go/tools/remotecommand"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	cio "github.com/containerd/containerd/pkg/cri/io"
+)
+
+// Attach prepares a streaming endpoint to attach to a running container, and returns the address.
+func (c *criService) Attach(ctx context.Context, r *runtime.AttachRequest) (*runtime.AttachResponse, error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container in store")
+	}
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+	return c.streamServer.GetAttach(r)
+}
+
+func (c *criService) attachContainer(ctx context.Context, id string, stdin io.Reader, stdout, stderr io.WriteCloser,
+	tty bool, resize <-chan remotecommand.TerminalSize) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	// Get container from our container store.
+	cntr, err := c.containerStore.Get(id)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find container %q in store", id)
+	}
+	id = cntr.ID
+
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+
+	task, err := cntr.Container.Task(ctx, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to load task")
+	}
+	handleResizing(ctx, resize, func(size remotecommand.TerminalSize) {
+		if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to resize task %q console", id)
+		}
+	})
+
+	opts := cio.AttachOptions{
+		Stdin:     stdin,
+		Stdout:    stdout,
+		Stderr:    stderr,
+		Tty:       tty,
+		StdinOnce: cntr.Config.StdinOnce,
+		CloseStdin: func() error {
+			return task.CloseIO(ctx, containerd.WithStdinCloser)
+		},
+	}
+	// TODO(random-liu): Figure out whether we need to support historical output.
+	cntr.IO.Attach(opts)
+	return nil
+}

pkg/cri/server/container_create.go

@@ -0,0 +1,343 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/typeurl"
+	"github.com/davecgh/go-spew/spew"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	cio "github.com/containerd/containerd/pkg/cri/io"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	"github.com/containerd/containerd/pkg/util"
+)
+
+func init() {
+	typeurl.Register(&containerstore.Metadata{},
+		"github.com/containerd/cri/pkg/store/container", "Metadata")
+}
+
+// CreateContainer creates a new container in the given PodSandbox.
+func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (_ *runtime.CreateContainerResponse, retErr error) {
+	config := r.GetConfig()
+	log.G(ctx).Debugf("Container config %+v", config)
+	sandboxConfig := r.GetSandboxConfig()
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find sandbox id %q", r.GetPodSandboxId())
+	}
+	sandboxID := sandbox.ID
+	s, err := sandbox.Container.Task(ctx, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox container task")
+	}
+	sandboxPid := s.Pid()
+
+	// Generate unique id and name for the container and reserve the name.
+	// Reserve the container name to avoid concurrent `CreateContainer` request creating
+	// the same container.
+	id := util.GenerateID()
+	metadata := config.GetMetadata()
+	if metadata == nil {
+		return nil, errors.New("container config must include metadata")
+	}
+	containerName := metadata.Name
+	name := makeContainerName(metadata, sandboxConfig.GetMetadata())
+	log.G(ctx).Debugf("Generated id %q for container %q", id, name)
+	if err = c.containerNameIndex.Reserve(name, id); err != nil {
+		return nil, errors.Wrapf(err, "failed to reserve container name %q", name)
+	}
+	defer func() {
+		// Release the name if the function returns with an error.
+		if retErr != nil {
+			c.containerNameIndex.ReleaseByName(name)
+		}
+	}()
+
+	// Create initial internal container metadata.
+	meta := containerstore.Metadata{
+		ID:        id,
+		Name:      name,
+		SandboxID: sandboxID,
+		Config:    config,
+	}
+
+	// Prepare container image snapshot. For container, the image should have
+	// been pulled before creating the container, so do not ensure the image.
+	image, err := c.localResolve(config.GetImage().GetImage())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to resolve image %q", config.GetImage().GetImage())
+	}
+	containerdImage, err := c.toContainerdImage(ctx, image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get image from containerd %q", image.ID)
+	}
+
+	// Run container using the same runtime with sandbox.
+	sandboxInfo, err := sandbox.Container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get sandbox %q info", sandboxID)
+	}
+
+	// Create container root directory.
+	containerRootDir := c.getContainerRootDir(id)
+	if err = c.os.MkdirAll(containerRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create container root directory %q",
+			containerRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the container root directory.
+			if err = c.os.RemoveAll(containerRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove container root directory %q",
+					containerRootDir)
+			}
+		}
+	}()
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err = c.os.MkdirAll(volatileContainerRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create volatile container root directory %q",
+			volatileContainerRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile container root directory.
+			if err = c.os.RemoveAll(volatileContainerRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove volatile container root directory %q",
+					volatileContainerRootDir)
+			}
+		}
+	}()
+
+	var volumeMounts []*runtime.Mount
+	if !c.config.IgnoreImageDefinedVolumes {
+		// Create container image volumes mounts.
+		volumeMounts = c.volumeMounts(containerRootDir, config.GetMounts(), &image.ImageSpec.Config)
+	} else if len(image.ImageSpec.Config.Volumes) != 0 {
+		log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID)
+	}
+
+	// Generate container mounts.
+	mounts := c.containerMounts(sandboxID, config)
+
+	ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox runtime")
+	}
+	log.G(ctx).Debugf("Use OCI runtime %+v for sandbox %q and container %q", ociRuntime, sandboxID, id)
+
+	spec, err := c.containerSpec(id, sandboxID, sandboxPid, sandbox.NetNSPath, containerName, config, sandboxConfig,
+		&image.ImageSpec.Config, append(mounts, volumeMounts...), ociRuntime)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to generate container %q spec", id)
+	}
+
+	meta.ProcessLabel = spec.Process.SelinuxLabel
+
+	// handle any KVM based runtime
+	if err := modifyProcessLabel(ociRuntime.Type, spec); err != nil {
+		return nil, err
+	}
+
+	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+		// If privileged don't set the SELinux label but still record it on the container so
+		// the unused MCS label can be release later
+		spec.Process.SelinuxLabel = ""
+	}
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(spec.Process.SelinuxLabel)
+		}
+	}()
+
+	log.G(ctx).Debugf("Container %q spec: %#+v", id, spew.NewFormatter(spec))
+
+	// Set snapshotter before any other options.
+	opts := []containerd.NewContainerOpts{
+		containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
+		// Prepare container rootfs. This is always writeable even if
+		// the container wants a readonly rootfs since we want to give
+		// the runtime (runc) a chance to modify (e.g. to create mount
+		// points corresponding to spec.Mounts) before making the
+		// rootfs readonly (requested by spec.Root.Readonly).
+		customopts.WithNewSnapshot(id, containerdImage),
+	}
+	if len(volumeMounts) > 0 {
+		mountMap := make(map[string]string)
+		for _, v := range volumeMounts {
+			mountMap[filepath.Clean(v.HostPath)] = v.ContainerPath
+		}
+		opts = append(opts, customopts.WithVolumes(mountMap))
+	}
+	meta.ImageRef = image.ID
+	meta.StopSignal = image.ImageSpec.Config.StopSignal
+
+	// Validate log paths and compose full container log path.
+	if sandboxConfig.GetLogDirectory() != "" && config.GetLogPath() != "" {
+		meta.LogPath = filepath.Join(sandboxConfig.GetLogDirectory(), config.GetLogPath())
+		log.G(ctx).Debugf("Composed container full log path %q using sandbox log dir %q and container log path %q",
+			meta.LogPath, sandboxConfig.GetLogDirectory(), config.GetLogPath())
+	} else {
+		log.G(ctx).Infof("Logging will be disabled due to empty log paths for sandbox (%q) or container (%q)",
+			sandboxConfig.GetLogDirectory(), config.GetLogPath())
+	}
+
+	containerIO, err := cio.NewContainerIO(id,
+		cio.WithNewFIFOs(volatileContainerRootDir, config.GetTty(), config.GetStdin()))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create container io")
+	}
+	defer func() {
+		if retErr != nil {
+			if err := containerIO.Close(); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to close container io %q", id)
+			}
+		}
+	}()
+
+	specOpts, err := c.containerSpecOpts(config, &image.ImageSpec.Config)
+	if err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+
+	containerLabels := buildLabels(config.Labels, containerKindContainer)
+
+	runtimeOptions, err := getRuntimeOptions(sandboxInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
+	opts = append(opts,
+		containerd.WithSpec(spec, specOpts...),
+		containerd.WithRuntime(sandboxInfo.Runtime.Name, runtimeOptions),
+		containerd.WithContainerLabels(containerLabels),
+		containerd.WithContainerExtension(containerMetadataExtension, &meta))
+	var cntr containerd.Container
+	if cntr, err = c.client.NewContainer(ctx, id, opts...); err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd container")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			if err := cntr.Delete(deferCtx, containerd.WithSnapshotCleanup); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd container %q", id)
+			}
+		}
+	}()
+
+	status := containerstore.Status{CreatedAt: time.Now().UnixNano()}
+	container, err := containerstore.NewContainer(meta,
+		containerstore.WithStatus(status, containerRootDir),
+		containerstore.WithContainer(cntr),
+		containerstore.WithContainerIO(containerIO),
+	)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create internal container object for %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup container checkpoint on error.
+			if err := container.Delete(); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to cleanup container checkpoint for %q", id)
+			}
+		}
+	}()
+
+	// Add container into container store.
+	if err := c.containerStore.Add(container); err != nil {
+		return nil, errors.Wrapf(err, "failed to add container %q into store", id)
+	}
+
+	return &runtime.CreateContainerResponse{ContainerId: id}, nil
+}
+
+// volumeMounts sets up image volumes for container. Rely on the removal of container
+// root directory to do cleanup. Note that image volume will be skipped, if there is criMounts
+// specified with the same destination.
+func (c *criService) volumeMounts(containerRootDir string, criMounts []*runtime.Mount, config *imagespec.ImageConfig) []*runtime.Mount {
+	if len(config.Volumes) == 0 {
+		return nil
+	}
+	var mounts []*runtime.Mount
+	for dst := range config.Volumes {
+		if isInCRIMounts(dst, criMounts) {
+			// Skip the image volume, if there is CRI defined volume mapping.
+			// TODO(random-liu): This should be handled by Kubelet in the future.
+			// Kubelet should decide what to use for image volume, and also de-duplicate
+			// the image volume and user mounts.
+			continue
+		}
+		volumeID := util.GenerateID()
+		src := filepath.Join(containerRootDir, "volumes", volumeID)
+		// addOCIBindMounts will create these volumes.
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath:  dst,
+			HostPath:       src,
+			SelinuxRelabel: true,
+		})
+	}
+	return mounts
+}
+
+// runtimeSpec returns a default runtime spec used in cri-containerd.
+func (c *criService) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
+	// GenerateSpec needs namespace.
+	ctx := ctrdutil.NamespacedContext()
+	container := &containers.Container{ID: id}
+
+	if baseSpecFile != "" {
+		baseSpec, ok := c.baseOCISpecs[baseSpecFile]
+		if !ok {
+			return nil, errors.Errorf("can't find base OCI spec %q", baseSpecFile)
+		}
+
+		spec := oci.Spec{}
+		if err := util.DeepCopy(&spec, &baseSpec); err != nil {
+			return nil, errors.Wrap(err, "failed to clone OCI spec")
+		}
+
+		// Fix up cgroups path
+		applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...)
+
+		if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil {
+			return nil, errors.Wrap(err, "failed to apply OCI options")
+		}
+
+		return &spec, nil
+	}
+
+	spec, err := oci.GenerateSpec(ctx, nil, container, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate spec")
+	}
+
+	return spec, nil
+}

pkg/cri/server/container_create_linux.go

@@ -0,0 +1,462 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/cgroups"
+	"github.com/containerd/containerd/contrib/apparmor"
+	"github.com/containerd/containerd/contrib/seccomp"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+	"github.com/containerd/containerd/pkg/cri/config"
+)
+
+const (
+	// profileNamePrefix is the prefix for loading profiles on a localhost. Eg. AppArmor localhost/profileName.
+	profileNamePrefix = "localhost/" // TODO (mikebrow): get localhost/ & runtime/default from CRI kubernetes/kubernetes#51747
+	// runtimeDefault indicates that we should use or create a runtime default profile.
+	runtimeDefault = "runtime/default"
+	// dockerDefault indicates that we should use or create a docker default profile.
+	dockerDefault = "docker/default"
+	// appArmorDefaultProfileName is name to use when creating a default apparmor profile.
+	appArmorDefaultProfileName = "cri-containerd.apparmor.d"
+	// unconfinedProfile is a string indicating one should run a pod/containerd without a security profile
+	unconfinedProfile = "unconfined"
+	// seccompDefaultProfile is the default seccomp profile.
+	seccompDefaultProfile = dockerDefault
+)
+
+// containerMounts sets up necessary container system file mounts
+// including /dev/shm, /etc/hosts and /etc/resolv.conf.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	var mounts []*runtime.Mount
+	securityContext := config.GetLinux().GetSecurityContext()
+	if !isInCRIMounts(etcHostname, config.GetMounts()) {
+		// /etc/hostname is added since 1.1.6, 1.2.4 and 1.3.
+		// For in-place upgrade, the old sandbox doesn't have the hostname file,
+		// do not mount this in that case.
+		// TODO(random-liu): Remove the check and always mount this when
+		// containerd 1.1 and 1.2 are deprecated.
+		hostpath := c.getSandboxHostname(sandboxID)
+		if _, err := c.os.Stat(hostpath); err == nil {
+			mounts = append(mounts, &runtime.Mount{
+				ContainerPath: etcHostname,
+				HostPath:      hostpath,
+				Readonly:      securityContext.GetReadonlyRootfs(),
+			})
+		}
+	}
+
+	if !isInCRIMounts(etcHosts, config.GetMounts()) {
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: etcHosts,
+			HostPath:      c.getSandboxHosts(sandboxID),
+			Readonly:      securityContext.GetReadonlyRootfs(),
+		})
+	}
+
+	// Mount sandbox resolv.config.
+	// TODO: Need to figure out whether we should always mount it as read-only
+	if !isInCRIMounts(resolvConfPath, config.GetMounts()) {
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: resolvConfPath,
+			HostPath:      c.getResolvPath(sandboxID),
+			Readonly:      securityContext.GetReadonlyRootfs(),
+		})
+	}
+
+	if !isInCRIMounts(devShm, config.GetMounts()) {
+		sandboxDevShm := c.getSandboxDevShm(sandboxID)
+		if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
+			sandboxDevShm = devShm
+		}
+		mounts = append(mounts, &runtime.Mount{
+			ContainerPath: devShm,
+			HostPath:      sandboxDevShm,
+			Readonly:      false,
+		})
+	}
+	return mounts
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (_ *runtimespec.Spec, retErr error) {
+
+	specOpts := []oci.SpecOpts{
+		customopts.WithoutRunMount,
+		customopts.WithoutDefaultSecuritySettings,
+		customopts.WithRelativeRoot(relativeRootfsPath),
+		customopts.WithProcessArgs(config, imageConfig),
+		oci.WithDefaultPathEnv,
+		// this will be set based on the security context below
+		oci.WithNewPrivileges,
+	}
+	if config.GetWorkingDir() != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir()))
+	} else if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if config.GetTty() {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+
+	// Add HOSTNAME env.
+	var (
+		err      error
+		hostname = sandboxConfig.GetHostname()
+	)
+	if hostname == "" {
+		if hostname, err = c.os.Hostname(); err != nil {
+			return nil, err
+		}
+	}
+	specOpts = append(specOpts, oci.WithEnv([]string{hostnameEnv + "=" + hostname}))
+
+	// Apply envs from image config first, so that envs from container config
+	// can override them.
+	env := imageConfig.Env
+	for _, e := range config.GetEnvs() {
+		env = append(env, e.GetKey()+"="+e.GetValue())
+	}
+	specOpts = append(specOpts, oci.WithEnv(env))
+
+	securityContext := config.GetLinux().GetSecurityContext()
+	labelOptions, err := toLabel(securityContext.GetSelinuxOptions())
+	if err != nil {
+		return nil, err
+	}
+	if len(labelOptions) == 0 {
+		// Use pod level SELinux config
+		if sandbox, err := c.sandboxStore.Get(sandboxID); err == nil {
+			labelOptions, err = selinux.DupSecOpt(sandbox.ProcessLabel)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	processLabel, mountLabel, err := label.InitLabels(labelOptions)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
+	}
+	defer func() {
+		if retErr != nil {
+			_ = label.ReleaseLabel(processLabel)
+		}
+	}()
+
+	specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel))
+
+	if !c.config.DisableProcMount {
+		// Apply masked paths if specified.
+		// If the container is privileged, this will be cleared later on.
+		if maskedPaths := securityContext.GetMaskedPaths(); maskedPaths != nil {
+			specOpts = append(specOpts, oci.WithMaskedPaths(maskedPaths))
+		}
+
+		// Apply readonly paths if specified.
+		// If the container is privileged, this will be cleared later on.
+		if readonlyPaths := securityContext.GetReadonlyPaths(); readonlyPaths != nil {
+			specOpts = append(specOpts, oci.WithReadonlyPaths(readonlyPaths))
+		}
+	}
+
+	if securityContext.GetPrivileged() {
+		if !sandboxConfig.GetLinux().GetSecurityContext().GetPrivileged() {
+			return nil, errors.New("no privileged container allowed in sandbox")
+		}
+		specOpts = append(specOpts, oci.WithPrivileged)
+		if !ociRuntime.PrivilegedWithoutHostDevices {
+			specOpts = append(specOpts, oci.WithHostDevices, oci.WithAllDevicesAllowed)
+		} else {
+			// add requested devices by the config as host devices are not automatically added
+			specOpts = append(specOpts, customopts.WithDevices(c.os, config), customopts.WithCapabilities(securityContext))
+		}
+	} else { // not privileged
+		specOpts = append(specOpts, customopts.WithDevices(c.os, config), customopts.WithCapabilities(securityContext))
+	}
+
+	// Clear all ambient capabilities. The implication of non-root + caps
+	// is not clearly defined in Kubernetes.
+	// See https://github.com/kubernetes/kubernetes/issues/56374
+	// Keep docker's behavior for now.
+	specOpts = append(specOpts,
+		customopts.WithoutAmbientCaps,
+		customopts.WithSelinuxLabels(processLabel, mountLabel),
+	)
+
+	// TODO: Figure out whether we should set no new privilege for sandbox container by default
+	if securityContext.GetNoNewPrivs() {
+		specOpts = append(specOpts, oci.WithNoNewPrivileges)
+	}
+	// TODO(random-liu): [P1] Set selinux options (privileged or not).
+	if securityContext.GetReadonlyRootfs() {
+		specOpts = append(specOpts, oci.WithRootFSReadonly())
+	}
+
+	if c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDisabledCgroups)
+	} else {
+		specOpts = append(specOpts, customopts.WithResources(config.GetLinux().GetResources(), c.config.TolerateMissingHugetlbController, c.config.DisableHugetlbController))
+		if sandboxConfig.GetLinux().GetCgroupParent() != "" {
+			cgroupsPath := getCgroupsPath(sandboxConfig.GetLinux().GetCgroupParent(), id)
+			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
+		}
+	}
+
+	supplementalGroups := securityContext.GetSupplementalGroups()
+
+	for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations,
+		ociRuntime.PodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		ociRuntime.ContainerAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithOOMScoreAdj(config, c.config.RestrictOOMScoreAdj),
+		customopts.WithPodNamespaces(securityContext, sandboxPid),
+		customopts.WithSupplementalGroups(supplementalGroups),
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer),
+		customopts.WithAnnotation(annotations.SandboxID, sandboxID),
+		customopts.WithAnnotation(annotations.ContainerName, containerName),
+	)
+	// cgroupns is used for hiding /sys/fs/cgroup from containers.
+	// For compatibility, cgroupns is not used when running in cgroup v1 mode or in privileged.
+	// https://github.com/containers/libpod/issues/4363
+	// https://github.com/kubernetes/enhancements/blob/0e409b47497e398b369c281074485c8de129694f/keps/sig-node/20191118-cgroups-v2.md#cgroup-namespace
+	if cgroups.Mode() == cgroups.Unified && !securityContext.GetPrivileged() {
+		specOpts = append(specOpts, oci.WithLinuxNamespace(
+			runtimespec.LinuxNamespace{
+				Type: runtimespec.CgroupNamespace,
+			}))
+	}
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
+}
+
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	var specOpts []oci.SpecOpts
+	securityContext := config.GetLinux().GetSecurityContext()
+	// Set container username. This could only be done by containerd, because it needs
+	// access to the container rootfs. Pass user name to containerd, and let it overwrite
+	// the spec for us.
+	userstr, err := generateUserString(
+		securityContext.GetRunAsUsername(),
+		securityContext.GetRunAsUser(),
+		securityContext.GetRunAsGroup())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate user string")
+	}
+	if userstr == "" {
+		// Lastly, since no user override was passed via CRI try to set via OCI
+		// Image
+		userstr = imageConfig.User
+	}
+	if userstr != "" {
+		specOpts = append(specOpts, oci.WithUser(userstr))
+	}
+
+	if securityContext.GetRunAsUsername() != "" {
+		userstr = securityContext.GetRunAsUsername()
+	} else {
+		// Even if RunAsUser is not set, we still call `GetValue` to get uid 0.
+		// Because it is still useful to get additional gids for uid 0.
+		userstr = strconv.FormatInt(securityContext.GetRunAsUser().GetValue(), 10)
+	}
+	specOpts = append(specOpts, customopts.WithAdditionalGIDs(userstr))
+
+	apparmorSpecOpts, err := generateApparmorSpecOpts(
+		securityContext.GetApparmorProfile(),
+		securityContext.GetPrivileged(),
+		c.apparmorEnabled())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate apparmor spec opts")
+	}
+	if apparmorSpecOpts != nil {
+		specOpts = append(specOpts, apparmorSpecOpts)
+	}
+
+	seccompSpecOpts, err := c.generateSeccompSpecOpts(
+		securityContext.GetSeccompProfilePath(),
+		securityContext.GetPrivileged(),
+		c.seccompEnabled())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate seccomp spec opts")
+	}
+	if seccompSpecOpts != nil {
+		specOpts = append(specOpts, seccompSpecOpts)
+	}
+	return specOpts, nil
+}
+
+// generateSeccompSpecOpts generates containerd SpecOpts for seccomp.
+func (c *criService) generateSeccompSpecOpts(seccompProf string, privileged, seccompEnabled bool) (oci.SpecOpts, error) {
+	if privileged {
+		// Do not set seccomp profile when container is privileged
+		return nil, nil
+	}
+	if seccompProf == "" {
+		seccompProf = c.config.UnsetSeccompProfile
+	}
+	// Set seccomp profile
+	if seccompProf == runtimeDefault || seccompProf == dockerDefault {
+		// use correct default profile (Eg. if not configured otherwise, the default is docker/default)
+		seccompProf = seccompDefaultProfile
+	}
+	if !seccompEnabled {
+		if seccompProf != "" && seccompProf != unconfinedProfile {
+			return nil, errors.New("seccomp is not supported")
+		}
+		return nil, nil
+	}
+	switch seccompProf {
+	case "", unconfinedProfile:
+		// Do not set seccomp profile.
+		return nil, nil
+	case dockerDefault:
+		// Note: WithDefaultProfile specOpts must be added after capabilities
+		return seccomp.WithDefaultProfile(), nil
+	default:
+		// Require and Trim default profile name prefix
+		if !strings.HasPrefix(seccompProf, profileNamePrefix) {
+			return nil, errors.Errorf("invalid seccomp profile %q", seccompProf)
+		}
+		return seccomp.WithProfile(strings.TrimPrefix(seccompProf, profileNamePrefix)), nil
+	}
+}
+
+// generateApparmorSpecOpts generates containerd SpecOpts for apparmor.
+func generateApparmorSpecOpts(apparmorProf string, privileged, apparmorEnabled bool) (oci.SpecOpts, error) {
+	if !apparmorEnabled {
+		// Should fail loudly if user try to specify apparmor profile
+		// but we don't support it.
+		if apparmorProf != "" && apparmorProf != unconfinedProfile {
+			return nil, errors.New("apparmor is not supported")
+		}
+		return nil, nil
+	}
+	switch apparmorProf {
+	// Based on kubernetes#51746, default apparmor profile should be applied
+	// for when apparmor is not specified.
+	case runtimeDefault, "":
+		if privileged {
+			// Do not set apparmor profile when container is privileged
+			return nil, nil
+		}
+		// TODO (mikebrow): delete created apparmor default profile
+		return apparmor.WithDefaultProfile(appArmorDefaultProfileName), nil
+	case unconfinedProfile:
+		return nil, nil
+	default:
+		// Require and Trim default profile name prefix
+		if !strings.HasPrefix(apparmorProf, profileNamePrefix) {
+			return nil, errors.Errorf("invalid apparmor profile %q", apparmorProf)
+		}
+		appArmorProfile := strings.TrimPrefix(apparmorProf, profileNamePrefix)
+		if profileExists, err := appArmorProfileExists(appArmorProfile); !profileExists {
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to generate apparmor spec opts")
+			}
+			return nil, errors.Errorf("apparmor profile not found %s", appArmorProfile)
+		}
+		return apparmor.WithProfile(appArmorProfile), nil
+	}
+}
+
+// appArmorProfileExists scans apparmor/profiles for the requested profile
+func appArmorProfileExists(profile string) (bool, error) {
+	if profile == "" {
+		return false, errors.New("nil apparmor profile is not supported")
+	}
+	profiles, err := os.Open("/sys/kernel/security/apparmor/profiles")
+	if err != nil {
+		return false, err
+	}
+	defer profiles.Close()
+
+	rbuff := bufio.NewReader(profiles)
+	for {
+		line, err := rbuff.ReadString('\n')
+		switch err {
+		case nil:
+			if strings.HasPrefix(line, profile+" (") {
+				return true, nil
+			}
+		case io.EOF:
+			return false, nil
+		default:
+			return false, err
+		}
+	}
+}
+
+// generateUserString generates valid user string based on OCI Image Spec
+// v1.0.0.
+//
+// CRI defines that the following combinations are valid:
+//
+// (none) -> ""
+// username -> username
+// username, uid -> username
+// username, uid, gid -> username:gid
+// username, gid -> username:gid
+// uid -> uid
+// uid, gid -> uid:gid
+// gid -> error
+//
+// TODO(random-liu): Add group name support in CRI.
+func generateUserString(username string, uid, gid *runtime.Int64Value) (string, error) {
+	var userstr, groupstr string
+	if uid != nil {
+		userstr = strconv.FormatInt(uid.GetValue(), 10)
+	}
+	if username != "" {
+		userstr = username
+	}
+	if gid != nil {
+		groupstr = strconv.FormatInt(gid.GetValue(), 10)
+	}
+	if userstr == "" {
+		if groupstr != "" {
+			return "", errors.Errorf("user group %q is specified without user", groupstr)
+		}
+		return "", nil
+	}
+	if groupstr != "" {
+		userstr = userstr + ":" + groupstr
+	}
+	return userstr, nil
+}

pkg/cri/server/container_create_other.go

@@ -0,0 +1,44 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/config"
+)
+
+// containerMounts sets up necessary container system file mounts
+// including /dev/shm, /etc/hosts and /etc/resolv.conf.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	return []*runtime.Mount{}
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (_ *runtimespec.Spec, retErr error) {
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec)
+}
+
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return []oci.SpecOpts{}, nil
+}

pkg/cri/server/container_create_other_test.go

@@ -0,0 +1,40 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// checkMount is defined by all tests but not used here
+var _ = checkMount
+
+func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig,
+	*imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) {
+	config := &runtime.ContainerConfig{}
+	sandboxConfig := &runtime.PodSandboxConfig{}
+	imageConfig := &imagespec.ImageConfig{}
+	specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) {
+	}
+	return config, sandboxConfig, imageConfig, specCheck
+}

pkg/cri/server/container_create_test.go

@@ -0,0 +1,407 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/constants"
+	"github.com/containerd/containerd/pkg/containerd/opts"
+	"github.com/containerd/containerd/pkg/cri/config"
+)
+
+func checkMount(t *testing.T, mounts []runtimespec.Mount, src, dest, typ string,
+	contains, notcontains []string) {
+	found := false
+	for _, m := range mounts {
+		if m.Source == src && m.Destination == dest {
+			assert.Equal(t, m.Type, typ)
+			for _, c := range contains {
+				assert.Contains(t, m.Options, c)
+			}
+			for _, n := range notcontains {
+				assert.NotContains(t, m.Options, n)
+			}
+			found = true
+			break
+		}
+	}
+	assert.True(t, found, "mount from %q to %q not found", src, dest)
+}
+
+func TestGeneralContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testPid := uint32(1234)
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	ociRuntime := config.Runtime{}
+	c := newTestCRIService()
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+	require.NoError(t, err)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+}
+
+func TestPodAnnotationPassthroughContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+
+	for desc, test := range map[string]struct {
+		podAnnotations []string
+		configChange   func(*runtime.PodSandboxConfig)
+		specCheck      func(*testing.T, *runtimespec.Spec)
+	}{
+		"a passthrough annotation should be passed as an OCI annotation": {
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+			},
+		},
+		"a non-passthrough annotation should not be passed as an OCI annotation": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["d"] = "e"
+			},
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+				_, ok := spec.Annotations["d"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations should support wildcard match": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["t.f"] = "j"
+				c.Annotations["z.g"] = "o"
+				c.Annotations["z"] = "o"
+				c.Annotations["y.ca"] = "b"
+				c.Annotations["y"] = "b"
+			},
+			podAnnotations: []string{"t*", "z.*", "y.c*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				t.Logf("%+v", spec.Annotations)
+				assert.Equal(t, spec.Annotations["t.f"], "j")
+				assert.Equal(t, spec.Annotations["z.g"], "o")
+				assert.Equal(t, spec.Annotations["y.ca"], "b")
+				_, ok := spec.Annotations["y"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["z"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			c := newTestCRIService()
+			containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+			if test.configChange != nil {
+				test.configChange(sandboxConfig)
+			}
+
+			ociRuntime := config.Runtime{
+				PodAnnotations: test.podAnnotations,
+			}
+			spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName,
+				containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+			assert.NoError(t, err)
+			assert.NotNil(t, spec)
+			specCheck(t, testID, testSandboxID, testPid, spec)
+			if test.specCheck != nil {
+				test.specCheck(t, spec)
+			}
+		})
+	}
+}
+
+func TestContainerSpecCommand(t *testing.T) {
+	for desc, test := range map[string]struct {
+		criEntrypoint   []string
+		criArgs         []string
+		imageEntrypoint []string
+		imageArgs       []string
+		expected        []string
+		expectErr       bool
+	}{
+		"should use cri entrypoint if it's specified": {
+			criEntrypoint:   []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"a", "b"},
+		},
+		"should use cri entrypoint if it's specified even if it's empty": {
+			criEntrypoint:   []string{},
+			criArgs:         []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"a", "b"},
+		},
+		"should use cri entrypoint and args if they are specified": {
+			criEntrypoint:   []string{"a", "b"},
+			criArgs:         []string{"c", "d"},
+			imageEntrypoint: []string{"e", "f"},
+			imageArgs:       []string{"g", "h"},
+			expected:        []string{"a", "b", "c", "d"},
+		},
+		"should use image entrypoint if cri entrypoint is not specified": {
+			criArgs:         []string{"a", "b"},
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"c", "d", "a", "b"},
+		},
+		"should use image args if both cri entrypoint and args are not specified": {
+			imageEntrypoint: []string{"c", "d"},
+			imageArgs:       []string{"e", "f"},
+			expected:        []string{"c", "d", "e", "f"},
+		},
+		"should return error if both entrypoint and args are empty": {
+			expectErr: true,
+		},
+	} {
+
+		config, _, imageConfig, _ := getCreateContainerTestData()
+		config.Command = test.criEntrypoint
+		config.Args = test.criArgs
+		imageConfig.Entrypoint = test.imageEntrypoint
+		imageConfig.Cmd = test.imageArgs
+
+		var spec runtimespec.Spec
+		err := opts.WithProcessArgs(config, imageConfig)(context.Background(), nil, nil, &spec)
+		if test.expectErr {
+			assert.Error(t, err)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, spec.Process.Args, desc)
+	}
+}
+
+func TestVolumeMounts(t *testing.T) {
+	testContainerRootDir := "test-container-root"
+	for desc, test := range map[string]struct {
+		criMounts         []*runtime.Mount
+		imageVolumes      map[string]struct{}
+		expectedMountDest []string
+	}{
+		"should setup rw mount for image volumes": {
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1": {},
+				"/test-volume-2": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-1",
+				"/test-volume-2",
+			},
+		},
+		"should skip image volumes if already mounted by CRI": {
+			criMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/test-volume-1",
+					HostPath:      "/test-hostpath-1",
+				},
+			},
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1": {},
+				"/test-volume-2": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-2",
+			},
+		},
+		"should compare and return cleanpath": {
+			criMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/test-volume-1",
+					HostPath:      "/test-hostpath-1",
+				},
+			},
+			imageVolumes: map[string]struct{}{
+				"/test-volume-1/": {},
+				"/test-volume-2/": {},
+			},
+			expectedMountDest: []string{
+				"/test-volume-2/",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		config := &imagespec.ImageConfig{
+			Volumes: test.imageVolumes,
+		}
+		c := newTestCRIService()
+		got := c.volumeMounts(testContainerRootDir, test.criMounts, config)
+		assert.Len(t, got, len(test.expectedMountDest))
+		for _, dest := range test.expectedMountDest {
+			found := false
+			for _, m := range got {
+				if m.ContainerPath == dest {
+					found = true
+					assert.Equal(t,
+						filepath.Dir(m.HostPath),
+						filepath.Join(testContainerRootDir, "volumes"))
+					break
+				}
+			}
+			assert.True(t, found)
+		}
+	}
+}
+
+func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+
+	for desc, test := range map[string]struct {
+		podAnnotations       []string
+		containerAnnotations []string
+		podConfigChange      func(*runtime.PodSandboxConfig)
+		configChange         func(*runtime.ContainerConfig)
+		specCheck            func(*testing.T, *runtimespec.Spec)
+	}{
+		"passthrough annotations from pod and container should be passed as an OCI annotation": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{"pod.annotation.1"},
+			containerAnnotations: []string{"container.annotation.1"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, "1", spec.Annotations["container.annotation.1"])
+				_, ok := spec.Annotations["container.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.3"]
+				assert.False(t, ok)
+				assert.Equal(t, "1", spec.Annotations["pod.annotation.1"])
+				_, ok = spec.Annotations["pod.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.3"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations from pod and container should support wildcard": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{"pod.annotation.*"},
+			containerAnnotations: []string{"container.annotation.*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, "1", spec.Annotations["container.annotation.1"])
+				assert.Equal(t, "2", spec.Annotations["container.annotation.2"])
+				assert.Equal(t, "3", spec.Annotations["container.annotation.3"])
+				assert.Equal(t, "1", spec.Annotations["pod.annotation.1"])
+				assert.Equal(t, "2", spec.Annotations["pod.annotation.2"])
+				assert.Equal(t, "3", spec.Annotations["pod.annotation.3"])
+			},
+		},
+		"annotations should not pass through if no passthrough annotations are configured": {
+			podConfigChange: func(p *runtime.PodSandboxConfig) {
+				p.Annotations["pod.annotation.1"] = "1"
+				p.Annotations["pod.annotation.2"] = "2"
+				p.Annotations["pod.annotation.3"] = "3"
+			},
+			configChange: func(c *runtime.ContainerConfig) {
+				c.Annotations["container.annotation.1"] = "1"
+				c.Annotations["container.annotation.2"] = "2"
+				c.Annotations["container.annotation.3"] = "3"
+			},
+			podAnnotations:       []string{},
+			containerAnnotations: []string{},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				_, ok := spec.Annotations["container.annotation.1"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["container.annotation.3"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.1"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.2"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["pod.annotation.3"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			c := newTestCRIService()
+			containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+			if test.configChange != nil {
+				test.configChange(containerConfig)
+			}
+			if test.podConfigChange != nil {
+				test.podConfigChange(sandboxConfig)
+			}
+			ociRuntime := config.Runtime{
+				PodAnnotations:       test.podAnnotations,
+				ContainerAnnotations: test.containerAnnotations,
+			}
+			spec, err := c.containerSpec(testID, testSandboxID, testPid, "", testContainerName,
+				containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
+			assert.NoError(t, err)
+			assert.NotNil(t, spec)
+			specCheck(t, testID, testSandboxID, testPid, spec)
+			if test.specCheck != nil {
+				test.specCheck(t, spec)
+			}
+		})
+	}
+}
+
+func TestBaseRuntimeSpec(t *testing.T) {
+	c := newTestCRIService()
+	c.baseOCISpecs = map[string]*oci.Spec{
+		"/etc/containerd/cri-base.json": {
+			Version:  "1.0.2",
+			Hostname: "old",
+		},
+	}
+
+	out, err := c.runtimeSpec("id1", "/etc/containerd/cri-base.json", oci.WithHostname("new"))
+	assert.NoError(t, err)
+
+	assert.Equal(t, "1.0.2", out.Version)
+	assert.Equal(t, "new", out.Hostname)
+
+	// Make sure original base spec not changed
+	assert.NotEqual(t, out, c.baseOCISpecs["/etc/containerd/cri-base.json"])
+	assert.Equal(t, c.baseOCISpecs["/etc/containerd/cri-base.json"].Hostname, "old")
+
+	assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
+}

pkg/cri/server/container_create_windows.go

@@ -0,0 +1,117 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+	"github.com/containerd/containerd/pkg/cri/config"
+)
+
+// No container mounts for windows.
+func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
+	return nil
+}
+
+func (c *criService) containerSpec(id string, sandboxID string, sandboxPid uint32, netNSPath string, containerName string,
+	config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig,
+	extraMounts []*runtime.Mount, ociRuntime config.Runtime) (*runtimespec.Spec, error) {
+	specOpts := []oci.SpecOpts{
+		customopts.WithProcessArgs(config, imageConfig),
+	}
+	if config.GetWorkingDir() != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(config.GetWorkingDir()))
+	} else if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if config.GetTty() {
+		specOpts = append(specOpts, oci.WithTTY)
+	}
+
+	// Apply envs from image config first, so that envs from container config
+	// can override them.
+	env := imageConfig.Env
+	for _, e := range config.GetEnvs() {
+		env = append(env, e.GetKey()+"="+e.GetValue())
+	}
+	specOpts = append(specOpts, oci.WithEnv(env))
+
+	specOpts = append(specOpts,
+		// Clear the root location since hcsshim expects it.
+		// NOTE: readonly rootfs doesn't work on windows.
+		customopts.WithoutRoot,
+		customopts.WithWindowsNetworkNamespace(netNSPath),
+		oci.WithHostname(sandboxConfig.GetHostname()),
+	)
+
+	specOpts = append(specOpts, customopts.WithWindowsMounts(c.os, config, extraMounts))
+
+	// Start with the image config user and override below if RunAsUsername is not "".
+	username := imageConfig.User
+
+	windowsConfig := config.GetWindows()
+	if windowsConfig != nil {
+		specOpts = append(specOpts, customopts.WithWindowsResources(windowsConfig.GetResources()))
+		securityCtx := windowsConfig.GetSecurityContext()
+		if securityCtx != nil {
+			runAsUser := securityCtx.GetRunAsUsername()
+			if runAsUser != "" {
+				username = runAsUser
+			}
+			cs := securityCtx.GetCredentialSpec()
+			if cs != "" {
+				specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs))
+			}
+		}
+	}
+
+	// There really isn't a good Windows way to verify that the username is available in the
+	// image as early as here like there is for Linux. Later on in the stack hcsshim
+	// will handle the behavior of erroring out if the user isn't available in the image
+	// when trying to run the init process.
+	specOpts = append(specOpts, oci.WithUser(username))
+
+	for pKey, pValue := range getPassthroughAnnotations(sandboxConfig.Annotations,
+		ociRuntime.PodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		ociRuntime.ContainerAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer),
+		customopts.WithAnnotation(annotations.SandboxID, sandboxID),
+		customopts.WithAnnotation(annotations.ContainerName, containerName),
+	)
+	return c.runtimeSpec(id, ociRuntime.BaseRuntimeSpec, specOpts...)
+}
+
+// No extra spec options needed for windows.
+func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return nil, nil
+}

pkg/cri/server/container_create_windows_test.go

@@ -0,0 +1,189 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	"github.com/containerd/containerd/pkg/cri/config"
+)
+
+func getCreateContainerTestData() (*runtime.ContainerConfig, *runtime.PodSandboxConfig,
+	*imagespec.ImageConfig, func(*testing.T, string, string, uint32, *runtimespec.Spec)) {
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image: &runtime.ImageSpec{
+			Image: "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		},
+		Command:    []string{"test", "command"},
+		Args:       []string{"test", "args"},
+		WorkingDir: "test-cwd",
+		Envs: []*runtime.KeyValue{
+			{Key: "k1", Value: "v1"},
+			{Key: "k2", Value: "v2"},
+			{Key: "k3", Value: "v3=v3bis"},
+			{Key: "k4", Value: "v4=v4bis=foop"},
+		},
+		Mounts: []*runtime.Mount{
+			// everything default
+			{
+				ContainerPath: "container-path-1",
+				HostPath:      "host-path-1",
+			},
+			// readOnly
+			{
+				ContainerPath: "container-path-2",
+				HostPath:      "host-path-2",
+				Readonly:      true,
+			},
+		},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+		Windows: &runtime.WindowsContainerConfig{
+			Resources: &runtime.WindowsContainerResources{
+				CpuShares:          100,
+				CpuCount:           200,
+				CpuMaximum:         300,
+				MemoryLimitInBytes: 400,
+			},
+			SecurityContext: &runtime.WindowsContainerSecurityContext{
+				RunAsUsername:  "test-user",
+				CredentialSpec: "{\"test\": \"spec\"}",
+			},
+		},
+	}
+	sandboxConfig := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-sandbox-name",
+			Uid:       "test-sandbox-uid",
+			Namespace: "test-sandbox-ns",
+			Attempt:   2,
+		},
+		Hostname:    "test-hostname",
+		Annotations: map[string]string{"c": "d"},
+	}
+	imageConfig := &imagespec.ImageConfig{
+		Env:        []string{"ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "ik4=iv4=iv4bis=boop"},
+		Entrypoint: []string{"/entrypoint"},
+		Cmd:        []string{"cmd"},
+		WorkingDir: "/workspace",
+		User:       "ContainerUser",
+	}
+	specCheck := func(t *testing.T, id string, sandboxID string, sandboxPid uint32, spec *runtimespec.Spec) {
+		assert.Nil(t, spec.Root)
+		assert.Equal(t, "test-hostname", spec.Hostname)
+		assert.Equal(t, []string{"test", "command", "test", "args"}, spec.Process.Args)
+		assert.Equal(t, "test-cwd", spec.Process.Cwd)
+		assert.Contains(t, spec.Process.Env, "k1=v1", "k2=v2", "k3=v3=v3bis", "ik4=iv4=iv4bis=boop")
+		assert.Contains(t, spec.Process.Env, "ik1=iv1", "ik2=iv2", "ik3=iv3=iv3bis", "k4=v4=v4bis=foop")
+
+		t.Logf("Check bind mount")
+		checkMount(t, spec.Mounts, "host-path-1", "container-path-1", "", []string{"rw"}, nil)
+		checkMount(t, spec.Mounts, "host-path-2", "container-path-2", "", []string{"ro"}, nil)
+
+		t.Logf("Check resource limits")
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, 100)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Count, 200)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Maximum, 300)
+		assert.EqualValues(t, *spec.Windows.Resources.Memory.Limit, 400)
+
+		// Also checks if override of the image configs user is behaving.
+		t.Logf("Check username")
+		assert.Contains(t, spec.Process.User.Username, "test-user")
+
+		t.Logf("Check credential spec")
+		assert.Contains(t, spec.Windows.CredentialSpec, "{\"test\": \"spec\"}")
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], sandboxID)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeContainer)
+	}
+	return config, sandboxConfig, imageConfig, specCheck
+}
+
+func TestContainerWindowsNetworkNamespace(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	assert.NotNil(t, spec.Windows)
+	assert.NotNil(t, spec.Windows.Network)
+	assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace)
+}
+
+func TestMountCleanPath(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	containerConfig.Mounts = append(containerConfig.Mounts, &runtime.Mount{
+		ContainerPath: "c:/test/container-path",
+		HostPath:      "c:/test/host-path",
+	})
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	checkMount(t, spec.Mounts, "c:\\test\\host-path", "c:\\test\\container-path", "", []string{"rw"}, nil)
+}
+
+func TestMountNamedPipe(t *testing.T) {
+	testID := "test-id"
+	testSandboxID := "sandbox-id"
+	testContainerName := "container-name"
+	testPid := uint32(1234)
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
+	containerConfig.Mounts = append(containerConfig.Mounts, &runtime.Mount{
+		ContainerPath: `\\.\pipe\foo`,
+		HostPath:      `\\.\pipe\foo`,
+	})
+	spec, err := c.containerSpec(testID, testSandboxID, testPid, nsPath, testContainerName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, testSandboxID, testPid, spec)
+	checkMount(t, spec.Mounts, `\\.\pipe\foo`, `\\.\pipe\foo`, "", []string{"rw"}, nil)
+}

pkg/cri/server/container_exec.go

@@ -0,0 +1,36 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// Exec prepares a streaming endpoint to execute a command in the container, and returns the address.
+func (c *criService) Exec(ctx context.Context, r *runtime.ExecRequest) (*runtime.ExecResponse, error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find container %q in store", r.GetContainerId())
+	}
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+	return c.streamServer.GetExec(r)
+}

pkg/cri/server/container_execsync.go

@@ -0,0 +1,211 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"bytes"
+	"io"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/oci"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"k8s.io/client-go/tools/remotecommand"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	cio "github.com/containerd/containerd/pkg/cri/io"
+	cioutil "github.com/containerd/containerd/pkg/ioutil"
+	"github.com/containerd/containerd/pkg/util"
+)
+
+// ExecSync executes a command in the container, and returns the stdout output.
+// If command exits with a non-zero exit code, an error is returned.
+func (c *criService) ExecSync(ctx context.Context, r *runtime.ExecSyncRequest) (*runtime.ExecSyncResponse, error) {
+	var stdout, stderr bytes.Buffer
+	exitCode, err := c.execInContainer(ctx, r.GetContainerId(), execOptions{
+		cmd:     r.GetCmd(),
+		stdout:  cioutil.NewNopWriteCloser(&stdout),
+		stderr:  cioutil.NewNopWriteCloser(&stderr),
+		timeout: time.Duration(r.GetTimeout()) * time.Second,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to exec in container")
+	}
+
+	return &runtime.ExecSyncResponse{
+		Stdout:   stdout.Bytes(),
+		Stderr:   stderr.Bytes(),
+		ExitCode: int32(*exitCode),
+	}, nil
+}
+
+// execOptions specifies how to execute command in container.
+type execOptions struct {
+	cmd     []string
+	stdin   io.Reader
+	stdout  io.WriteCloser
+	stderr  io.WriteCloser
+	tty     bool
+	resize  <-chan remotecommand.TerminalSize
+	timeout time.Duration
+}
+
+func (c *criService) execInternal(ctx context.Context, container containerd.Container, id string, opts execOptions) (*uint32, error) {
+	// Cancel the context before returning to ensure goroutines are stopped.
+	// This is important, because if `Start` returns error, `Wait` will hang
+	// forever unless we cancel the context.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	spec, err := container.Spec(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container spec")
+	}
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load task")
+	}
+	pspec := spec.Process
+
+	pspec.Terminal = opts.tty
+	if opts.tty {
+		if err := oci.WithEnv([]string{"TERM=xterm"})(ctx, nil, nil, spec); err != nil {
+			return nil, errors.Wrap(err, "add TERM env var to spec")
+		}
+	}
+
+	pspec.Args = opts.cmd
+
+	if opts.stdout == nil {
+		opts.stdout = cio.NewDiscardLogger()
+	}
+	if opts.stderr == nil {
+		opts.stderr = cio.NewDiscardLogger()
+	}
+	execID := util.GenerateID()
+	log.G(ctx).Debugf("Generated exec id %q for container %q", execID, id)
+	volatileRootDir := c.getVolatileContainerRootDir(id)
+	var execIO *cio.ExecIO
+	process, err := task.Exec(ctx, execID, pspec,
+		func(id string) (containerdio.IO, error) {
+			var err error
+			execIO, err = cio.NewExecIO(id, volatileRootDir, opts.tty, opts.stdin != nil)
+			return execIO, err
+		},
+	)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create exec %q", execID)
+	}
+	defer func() {
+		deferCtx, deferCancel := ctrdutil.DeferContext()
+		defer deferCancel()
+		if _, err := process.Delete(deferCtx, containerd.WithProcessKill); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to delete exec process %q for container %q", execID, id)
+		}
+	}()
+
+	exitCh, err := process.Wait(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to wait for process %q", execID)
+	}
+	if err := process.Start(ctx); err != nil {
+		return nil, errors.Wrapf(err, "failed to start exec %q", execID)
+	}
+
+	handleResizing(ctx, opts.resize, func(size remotecommand.TerminalSize) {
+		if err := process.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to resize process %q console for container %q", execID, id)
+		}
+	})
+
+	attachDone := execIO.Attach(cio.AttachOptions{
+		Stdin:     opts.stdin,
+		Stdout:    opts.stdout,
+		Stderr:    opts.stderr,
+		Tty:       opts.tty,
+		StdinOnce: true,
+		CloseStdin: func() error {
+			return process.CloseIO(ctx, containerd.WithStdinCloser)
+		},
+	})
+
+	execCtx := ctx
+	if opts.timeout > 0 {
+		var execCtxCancel context.CancelFunc
+		execCtx, execCtxCancel = context.WithTimeout(ctx, opts.timeout)
+		defer execCtxCancel()
+	}
+
+	select {
+	case <-execCtx.Done():
+		// Ignore the not found error because the process may exit itself before killing.
+		if err := process.Kill(ctx, syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to kill exec %q", execID)
+		}
+		// Wait for the process to be killed.
+		exitRes := <-exitCh
+		log.G(ctx).Infof("Timeout received while waiting for exec process kill %q code %d and error %v",
+			execID, exitRes.ExitCode(), exitRes.Error())
+		<-attachDone
+		log.G(ctx).Debugf("Stream pipe for exec process %q done", execID)
+		return nil, errors.Wrapf(execCtx.Err(), "timeout %v exceeded", opts.timeout)
+	case exitRes := <-exitCh:
+		code, _, err := exitRes.Result()
+		log.G(ctx).Infof("Exec process %q exits with exit code %d and error %v", execID, code, err)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed while waiting for exec %q", execID)
+		}
+		<-attachDone
+		log.G(ctx).Debugf("Stream pipe for exec process %q done", execID)
+		return &code, nil
+	}
+}
+
+// execInContainer executes a command inside the container synchronously, and
+// redirects stdio stream properly.
+// This function only returns when the exec process exits, this means that:
+// 1) As long as the exec process is running, the goroutine in the cri plugin
+// will be running and wait for the exit code;
+// 2) `kubectl exec -it` will hang until the exec process exits, even after io
+// is detached. This is different from dockershim, which leaves the exec process
+// running in background after io is detached.
+// https://github.com/kubernetes/kubernetes/blob/v1.15.0/pkg/kubelet/dockershim/exec.go#L127
+// For example, if the `kubectl exec -it` process is killed, IO will be closed. In
+// this case, the CRI plugin will still have a goroutine waiting for the exec process
+// to exit and log the exit code, but dockershim won't.
+func (c *criService) execInContainer(ctx context.Context, id string, opts execOptions) (*uint32, error) {
+	// Get container from our container store.
+	cntr, err := c.containerStore.Get(id)
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find container %q in store", id)
+	}
+	id = cntr.ID
+
+	state := cntr.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.Errorf("container is in %s state", criContainerStateToString(state))
+	}
+
+	return c.execInternal(ctx, cntr.Container, id, opts)
+}

pkg/cri/server/container_list.go

@@ -0,0 +1,112 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// ListContainers lists all containers matching the filter.
+func (c *criService) ListContainers(ctx context.Context, r *runtime.ListContainersRequest) (*runtime.ListContainersResponse, error) {
+	// List all containers from store.
+	containersInStore := c.containerStore.List()
+
+	var containers []*runtime.Container
+	for _, container := range containersInStore {
+		containers = append(containers, toCRIContainer(container))
+	}
+
+	containers = c.filterCRIContainers(containers, r.GetFilter())
+	return &runtime.ListContainersResponse{Containers: containers}, nil
+}
+
+// toCRIContainer converts internal container object into CRI container.
+func toCRIContainer(container containerstore.Container) *runtime.Container {
+	status := container.Status.Get()
+	return &runtime.Container{
+		Id:           container.ID,
+		PodSandboxId: container.SandboxID,
+		Metadata:     container.Config.GetMetadata(),
+		Image:        container.Config.GetImage(),
+		ImageRef:     container.ImageRef,
+		State:        status.State(),
+		CreatedAt:    status.CreatedAt,
+		Labels:       container.Config.GetLabels(),
+		Annotations:  container.Config.GetAnnotations(),
+	}
+}
+
+func (c *criService) normalizeContainerFilter(filter *runtime.ContainerFilter) {
+	if cntr, err := c.containerStore.Get(filter.GetId()); err == nil {
+		filter.Id = cntr.ID
+	}
+	if sb, err := c.sandboxStore.Get(filter.GetPodSandboxId()); err == nil {
+		filter.PodSandboxId = sb.ID
+	}
+}
+
+// filterCRIContainers filters CRIContainers.
+func (c *criService) filterCRIContainers(containers []*runtime.Container, filter *runtime.ContainerFilter) []*runtime.Container {
+	if filter == nil {
+		return containers
+	}
+
+	// The containerd cri plugin supports short ids so long as there is only one
+	// match. So we do a lookup against the store here if a pod id has been
+	// included in the filter.
+	sb := filter.GetPodSandboxId()
+	if sb != "" {
+		sandbox, err := c.sandboxStore.Get(sb)
+		if err == nil {
+			sb = sandbox.ID
+		}
+	}
+
+	c.normalizeContainerFilter(filter)
+	filtered := []*runtime.Container{}
+	for _, cntr := range containers {
+		if filter.GetId() != "" && filter.GetId() != cntr.Id {
+			continue
+		}
+		if sb != "" && sb != cntr.PodSandboxId {
+			continue
+		}
+		if filter.GetState() != nil && filter.GetState().GetState() != cntr.State {
+			continue
+		}
+		if filter.GetLabelSelector() != nil {
+			match := true
+			for k, v := range filter.GetLabelSelector() {
+				got, ok := cntr.Labels[k]
+				if !ok || got != v {
+					match = false
+					break
+				}
+			}
+			if !match {
+				continue
+			}
+		}
+		filtered = append(filtered, cntr)
+	}
+
+	return filtered
+}

pkg/cri/server/container_list_test.go

@@ -0,0 +1,345 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestToCRIContainer(t *testing.T) {
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image:       &runtime.ImageSpec{Image: "test-image"},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+	createdAt := time.Now().UnixNano()
+	container, err := containerstore.NewContainer(
+		containerstore.Metadata{
+			ID:        "test-id",
+			Name:      "test-name",
+			SandboxID: "test-sandbox-id",
+			Config:    config,
+			ImageRef:  "test-image-ref",
+		},
+		containerstore.WithFakeStatus(
+			containerstore.Status{
+				Pid:        1234,
+				CreatedAt:  createdAt,
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+				ExitCode:   1,
+				Reason:     "test-reason",
+				Message:    "test-message",
+			},
+		),
+	)
+	assert.NoError(t, err)
+	expect := &runtime.Container{
+		Id:           "test-id",
+		PodSandboxId: "test-sandbox-id",
+		Metadata:     config.GetMetadata(),
+		Image:        config.GetImage(),
+		ImageRef:     "test-image-ref",
+		State:        runtime.ContainerState_CONTAINER_EXITED,
+		CreatedAt:    createdAt,
+		Labels:       config.GetLabels(),
+		Annotations:  config.GetAnnotations(),
+	}
+	c := toCRIContainer(container)
+	assert.Equal(t, expect, c)
+}
+
+func TestFilterContainers(t *testing.T) {
+	c := newTestCRIService()
+
+	testContainers := []*runtime.Container{
+		{
+			Id:           "1",
+			PodSandboxId: "s-1",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-1", Attempt: 1},
+			State:        runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		{
+			Id:           "2",
+			PodSandboxId: "s-2",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2", Attempt: 2},
+			State:        runtime.ContainerState_CONTAINER_EXITED,
+			Labels:       map[string]string{"a": "b"},
+		},
+		{
+			Id:           "3",
+			PodSandboxId: "s-2",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2", Attempt: 3},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			Labels:       map[string]string{"c": "d"},
+		},
+	}
+	for desc, test := range map[string]struct {
+		filter *runtime.ContainerFilter
+		expect []*runtime.Container
+	}{
+		"no filter": {
+			expect: testContainers,
+		},
+		"id filter": {
+			filter: &runtime.ContainerFilter{Id: "2"},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"state filter": {
+			filter: &runtime.ContainerFilter{
+				State: &runtime.ContainerStateValue{
+					State: runtime.ContainerState_CONTAINER_EXITED,
+				},
+			},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"label filter": {
+			filter: &runtime.ContainerFilter{
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.Container{testContainers[1]},
+		},
+		"sandbox id filter": {
+			filter: &runtime.ContainerFilter{PodSandboxId: "s-2"},
+			expect: []*runtime.Container{testContainers[1], testContainers[2]},
+		},
+		"mixed filter not matched": {
+			filter: &runtime.ContainerFilter{
+				Id:            "1",
+				PodSandboxId:  "s-2",
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.Container{},
+		},
+		"mixed filter matched": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-2",
+				State: &runtime.ContainerStateValue{
+					State: runtime.ContainerState_CONTAINER_CREATED,
+				},
+				LabelSelector: map[string]string{"c": "d"},
+			},
+			expect: []*runtime.Container{testContainers[2]},
+		},
+	} {
+		filtered := c.filterCRIContainers(testContainers, test.filter)
+		assert.Equal(t, test.expect, filtered, desc)
+	}
+}
+
+// containerForTest is a helper type for test.
+type containerForTest struct {
+	metadata containerstore.Metadata
+	status   containerstore.Status
+}
+
+func (c containerForTest) toContainer() (containerstore.Container, error) {
+	return containerstore.NewContainer(
+		c.metadata,
+		containerstore.WithFakeStatus(c.status),
+	)
+}
+
+func TestListContainers(t *testing.T) {
+	c := newTestCRIService()
+	sandboxesInStore := []sandboxstore.Sandbox{
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:     "s-1abcdef1234",
+				Name:   "sandboxname-1",
+				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "podname-1"}},
+			},
+			sandboxstore.Status{
+				State: sandboxstore.StateReady,
+			},
+		),
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:     "s-2abcdef1234",
+				Name:   "sandboxname-2",
+				Config: &runtime.PodSandboxConfig{Metadata: &runtime.PodSandboxMetadata{Name: "podname-2"}},
+			},
+			sandboxstore.Status{
+				State: sandboxstore.StateNotReady,
+			},
+		),
+	}
+	createdAt := time.Now().UnixNano()
+	startedAt := time.Now().UnixNano()
+	finishedAt := time.Now().UnixNano()
+	containersInStore := []containerForTest{
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-1container",
+				Name:      "name-1",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-1"}},
+			},
+			status: containerstore.Status{CreatedAt: createdAt},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-2container",
+				Name:      "name-2",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-2"}},
+			},
+			status: containerstore.Status{
+				CreatedAt: createdAt,
+				StartedAt: startedAt,
+			},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-3container",
+				Name:      "name-3",
+				SandboxID: "s-1abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-3"}},
+			},
+			status: containerstore.Status{
+				CreatedAt:  createdAt,
+				StartedAt:  startedAt,
+				FinishedAt: finishedAt,
+			},
+		},
+		{
+			metadata: containerstore.Metadata{
+				ID:        "c-4container",
+				Name:      "name-4",
+				SandboxID: "s-2abcdef1234",
+				Config:    &runtime.ContainerConfig{Metadata: &runtime.ContainerMetadata{Name: "name-4"}},
+			},
+			status: containerstore.Status{
+				CreatedAt: createdAt,
+			},
+		},
+	}
+
+	expectedContainers := []*runtime.Container{
+		{
+			Id:           "c-1container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-1"},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-2container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-2"},
+			State:        runtime.ContainerState_CONTAINER_RUNNING,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-3container",
+			PodSandboxId: "s-1abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-3"},
+			State:        runtime.ContainerState_CONTAINER_EXITED,
+			CreatedAt:    createdAt,
+		},
+		{
+			Id:           "c-4container",
+			PodSandboxId: "s-2abcdef1234",
+			Metadata:     &runtime.ContainerMetadata{Name: "name-4"},
+			State:        runtime.ContainerState_CONTAINER_CREATED,
+			CreatedAt:    createdAt,
+		},
+	}
+
+	// Inject test sandbox metadata
+	for _, sb := range sandboxesInStore {
+		assert.NoError(t, c.sandboxStore.Add(sb))
+	}
+
+	// Inject test container metadata
+	for _, cntr := range containersInStore {
+		container, err := cntr.toContainer()
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(container))
+	}
+
+	for testdesc, testdata := range map[string]struct {
+		filter *runtime.ContainerFilter
+		expect []*runtime.Container
+	}{
+		"test without filter": {
+			filter: &runtime.ContainerFilter{},
+			expect: expectedContainers,
+		},
+		"test filter by sandboxid": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-1abcdef1234",
+			},
+			expect: expectedContainers[:3],
+		},
+		"test filter by truncated sandboxid": {
+			filter: &runtime.ContainerFilter{
+				PodSandboxId: "s-1",
+			},
+			expect: expectedContainers[:3],
+		},
+		"test filter by containerid": {
+			filter: &runtime.ContainerFilter{
+				Id: "c-1container",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by truncated containerid": {
+			filter: &runtime.ContainerFilter{
+				Id: "c-1",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by containerid and sandboxid": {
+			filter: &runtime.ContainerFilter{
+				Id:           "c-1container",
+				PodSandboxId: "s-1abcdef1234",
+			},
+			expect: expectedContainers[:1],
+		},
+		"test filter by truncated containerid and truncated sandboxid": {
+			filter: &runtime.ContainerFilter{
+				Id:           "c-1",
+				PodSandboxId: "s-1",
+			},
+			expect: expectedContainers[:1],
+		},
+	} {
+		t.Logf("TestCase: %s", testdesc)
+		resp, err := c.ListContainers(context.Background(), &runtime.ListContainersRequest{Filter: testdata.filter})
+		assert.NoError(t, err)
+		require.NotNil(t, resp)
+		containers := resp.GetContainers()
+		assert.Len(t, containers, len(testdata.expect))
+		for _, cntr := range testdata.expect {
+			assert.Contains(t, containers, cntr)
+		}
+	}
+}

pkg/cri/server/container_log_reopen.go

@@ -0,0 +1,51 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ReopenContainerLog asks the cri plugin to reopen the stdout/stderr log file for the container.
+// This is often called after the log file has been rotated.
+func (c *criService) ReopenContainerLog(ctx context.Context, r *runtime.ReopenContainerLogRequest) (*runtime.ReopenContainerLogResponse, error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	if container.Status.Get().State() != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil, errors.New("container is not running")
+	}
+
+	// Create new container logger and replace the existing ones.
+	stdoutWC, stderrWC, err := c.createContainerLoggers(container.LogPath, container.Config.GetTty())
+	if err != nil {
+		return nil, err
+	}
+	oldStdoutWC, oldStderrWC := container.IO.AddOutput("log", stdoutWC, stderrWC)
+	if oldStdoutWC != nil {
+		oldStdoutWC.Close()
+	}
+	if oldStderrWC != nil {
+		oldStderrWC.Close()
+	}
+	return &runtime.ReopenContainerLogResponse{}, nil
+}

pkg/cri/server/container_remove.go

@@ -0,0 +1,135 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// RemoveContainer removes the container.
+func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveContainerRequest) (_ *runtime.RemoveContainerResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+		}
+		// Do not return error if container metadata doesn't exist.
+		log.G(ctx).Tracef("RemoveContainer called for container %q that does not exist", r.GetContainerId())
+		return &runtime.RemoveContainerResponse{}, nil
+	}
+	id := container.ID
+
+	// Forcibly stop the containers if they are in running or unknown state
+	state := container.Status.Get().State()
+	if state == runtime.ContainerState_CONTAINER_RUNNING ||
+		state == runtime.ContainerState_CONTAINER_UNKNOWN {
+		logrus.Infof("Forcibly stopping container %q", id)
+		if err := c.stopContainer(ctx, container, 0); err != nil {
+			return nil, errors.Wrapf(err, "failed to forcibly stop container %q", id)
+		}
+
+	}
+
+	// Set removing state to prevent other start/remove operations against this container
+	// while it's being removed.
+	if err := setContainerRemoving(container); err != nil {
+		return nil, errors.Wrapf(err, "failed to set removing state for container %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Reset removing if remove failed.
+			if err := resetContainerRemoving(container); err != nil {
+				log.G(ctx).WithError(err).Errorf("failed to reset removing state for container %q", id)
+			}
+		}
+	}()
+
+	// NOTE(random-liu): Docker set container to "Dead" state when start removing the
+	// container so as to avoid start/restart the container again. However, for current
+	// kubelet implementation, we'll never start a container once we decide to remove it,
+	// so we don't need the "Dead" state for now.
+
+	// Delete containerd container.
+	if err := container.Container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
+		if !errdefs.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to delete containerd container %q", id)
+		}
+		log.G(ctx).Tracef("Remove called for containerd container %q that does not exist", id)
+	}
+
+	// Delete container checkpoint.
+	if err := container.Delete(); err != nil {
+		return nil, errors.Wrapf(err, "failed to delete container checkpoint for %q", id)
+	}
+
+	containerRootDir := c.getContainerRootDir(id)
+	if err := ensureRemoveAll(ctx, containerRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove container root directory %q",
+			containerRootDir)
+	}
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err := ensureRemoveAll(ctx, volatileContainerRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove volatile container root directory %q",
+			volatileContainerRootDir)
+	}
+
+	c.containerStore.Delete(id)
+
+	c.containerNameIndex.ReleaseByKey(id)
+
+	return &runtime.RemoveContainerResponse{}, nil
+}
+
+// setContainerRemoving sets the container into removing state. In removing state, the
+// container will not be started or removed again.
+func setContainerRemoving(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		// Do not remove container if it's still running or unknown.
+		if status.State() == runtime.ContainerState_CONTAINER_RUNNING {
+			return status, errors.New("container is still running, to stop first")
+		}
+		if status.State() == runtime.ContainerState_CONTAINER_UNKNOWN {
+			return status, errors.New("container state is unknown, to stop first")
+		}
+		if status.Starting {
+			return status, errors.New("container is in starting state, can't be removed")
+		}
+		if status.Removing {
+			return status, errors.New("container is already in removing state")
+		}
+		status.Removing = true
+		return status, nil
+	})
+}
+
+// resetContainerRemoving resets the container removing state on remove failure. So
+// that we could remove the container again.
+func resetContainerRemoving(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Removing = false
+		return status, nil
+	})
+}

pkg/cri/server/container_remove_test.go

@@ -0,0 +1,85 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// TestSetContainerRemoving tests setContainerRemoving sets removing
+// state correctly.
+func TestSetContainerRemoving(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		status    containerstore.Status
+		expectErr bool
+	}{
+		"should return error when container is in running state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in starting state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Starting:  true,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in removing state": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+				Removing:   true,
+			},
+			expectErr: true,
+		},
+		"should not return error when container is not running and removing": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			expectErr: false,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: testID},
+			containerstore.WithFakeStatus(test.status),
+		)
+		assert.NoError(t, err)
+		err = setContainerRemoving(container)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Equal(t, test.status, container.Status.Get(), "metadata should not be updated")
+		} else {
+			assert.NoError(t, err)
+			assert.True(t, container.Status.Get().Removing, "removing should be set")
+			assert.NoError(t, resetContainerRemoving(container))
+			assert.False(t, container.Status.Get().Removing, "removing should be reset")
+		}
+	}
+}

pkg/cri/server/container_start.go

@@ -0,0 +1,223 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io"
+	"time"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	cio "github.com/containerd/containerd/pkg/cri/io"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	cioutil "github.com/containerd/containerd/pkg/ioutil"
+)
+
+// StartContainer starts the container.
+func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) {
+	cntr, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	id := cntr.ID
+	meta := cntr.Metadata
+	container := cntr.Container
+	config := meta.Config
+
+	// Set starting state to prevent other start/remove operations against this container
+	// while it's being started.
+	if err := setContainerStarting(cntr); err != nil {
+		return nil, errors.Wrapf(err, "failed to set starting state for container %q", id)
+	}
+	defer func() {
+		if retErr != nil {
+			// Set container to exited if fail to start.
+			if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+				status.Pid = 0
+				status.FinishedAt = time.Now().UnixNano()
+				status.ExitCode = errorStartExitCode
+				status.Reason = errorStartReason
+				status.Message = retErr.Error()
+				return status, nil
+			}); err != nil {
+				log.G(ctx).WithError(err).Errorf("failed to set start failure state for container %q", id)
+			}
+		}
+		if err := resetContainerStarting(cntr); err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to reset starting state for container %q", id)
+		}
+	}()
+
+	// Get sandbox config from sandbox store.
+	sandbox, err := c.sandboxStore.Get(meta.SandboxID)
+	if err != nil {
+		return nil, errors.Wrapf(err, "sandbox %q not found", meta.SandboxID)
+	}
+	sandboxID := meta.SandboxID
+	if sandbox.Status.Get().State != sandboxstore.StateReady {
+		return nil, errors.Errorf("sandbox container %q is not running", sandboxID)
+	}
+
+	ioCreation := func(id string) (_ containerdio.IO, err error) {
+		stdoutWC, stderrWC, err := c.createContainerLoggers(meta.LogPath, config.GetTty())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create container loggers")
+		}
+		cntr.IO.AddOutput("log", stdoutWC, stderrWC)
+		cntr.IO.Pipe()
+		return cntr.IO, nil
+	}
+
+	ctrInfo, err := container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container info")
+	}
+
+	taskOpts := c.taskOpts(ctrInfo.Runtime.Name)
+	task, err := container.NewTask(ctx, ioCreation, taskOpts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd task")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// It's possible that task is deleted by event monitor.
+			if _, err := task.Delete(deferCtx, WithNRISandboxDelete(sandboxID), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd task %q", id)
+			}
+		}
+	}()
+
+	// wait is a long running background request, no timeout needed.
+	exitCh, err := task.Wait(ctrdutil.NamespacedContext())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to wait for containerd task")
+	}
+	nric, err := nri.New()
+	if err != nil {
+		log.G(ctx).WithError(err).Error("unable to create nri client")
+	}
+	if nric != nil {
+		nriSB := &nri.Sandbox{
+			ID:     sandboxID,
+			Labels: sandbox.Config.Labels,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil {
+			return nil, errors.Wrap(err, "nri invoke")
+		}
+	}
+
+	// Start containerd task.
+	if err := task.Start(ctx); err != nil {
+		return nil, errors.Wrapf(err, "failed to start containerd task %q", id)
+	}
+
+	// Update container start timestamp.
+	if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Pid = task.Pid()
+		status.StartedAt = time.Now().UnixNano()
+		return status, nil
+	}); err != nil {
+		return nil, errors.Wrapf(err, "failed to update container %q state", id)
+	}
+
+	// start the monitor after updating container state, this ensures that
+	// event monitor receives the TaskExit event and update container state
+	// after this.
+	c.eventMonitor.startExitMonitor(context.Background(), id, task.Pid(), exitCh)
+
+	return &runtime.StartContainerResponse{}, nil
+}
+
+// setContainerStarting sets the container into starting state. In starting state, the
+// container will not be removed or started again.
+func setContainerStarting(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		// Return error if container is not in created state.
+		if status.State() != runtime.ContainerState_CONTAINER_CREATED {
+			return status, errors.Errorf("container is in %s state", criContainerStateToString(status.State()))
+		}
+		// Do not start the container when there is a removal in progress.
+		if status.Removing {
+			return status, errors.New("container is in removing state, can't be started")
+		}
+		if status.Starting {
+			return status, errors.New("container is already in starting state")
+		}
+		status.Starting = true
+		return status, nil
+	})
+}
+
+// resetContainerStarting resets the container starting state on start failure. So
+// that we could remove the container later.
+func resetContainerStarting(container containerstore.Container) error {
+	return container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		status.Starting = false
+		return status, nil
+	})
+}
+
+// createContainerLoggers creates container loggers and return write closer for stdout and stderr.
+func (c *criService) createContainerLoggers(logPath string, tty bool) (stdout io.WriteCloser, stderr io.WriteCloser, err error) {
+	if logPath != "" {
+		// Only generate container log when log path is specified.
+		f, err := openLogFile(logPath)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "failed to create and open log file")
+		}
+		defer func() {
+			if err != nil {
+				f.Close()
+			}
+		}()
+		var stdoutCh, stderrCh <-chan struct{}
+		wc := cioutil.NewSerialWriteCloser(f)
+		stdout, stdoutCh = cio.NewCRILogger(logPath, wc, cio.Stdout, c.config.MaxContainerLogLineSize)
+		// Only redirect stderr when there is no tty.
+		if !tty {
+			stderr, stderrCh = cio.NewCRILogger(logPath, wc, cio.Stderr, c.config.MaxContainerLogLineSize)
+		}
+		go func() {
+			if stdoutCh != nil {
+				<-stdoutCh
+			}
+			if stderrCh != nil {
+				<-stderrCh
+			}
+			logrus.Debugf("Finish redirecting log file %q, closing it", logPath)
+			f.Close()
+		}()
+	} else {
+		stdout = cio.NewDiscardLogger()
+		stderr = cio.NewDiscardLogger()
+	}
+	return
+}

pkg/cri/server/container_start_test.go

@@ -0,0 +1,98 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// TestSetContainerStarting tests setContainerStarting sets removing
+// state correctly.
+func TestSetContainerStarting(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		status    containerstore.Status
+		expectErr bool
+	}{
+
+		"should not return error when container is in created state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+			},
+			expectErr: false,
+		},
+		"should return error when container is in running state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in exited state": {
+			status: containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			expectErr: true,
+		},
+		"should return error when container is in unknown state": {
+			status: containerstore.Status{
+				CreatedAt:  0,
+				StartedAt:  0,
+				FinishedAt: 0,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in starting state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Starting:  true,
+			},
+			expectErr: true,
+		},
+		"should return error when container is in removing state": {
+			status: containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				Removing:  true,
+			},
+			expectErr: true,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: testID},
+			containerstore.WithFakeStatus(test.status),
+		)
+		assert.NoError(t, err)
+		err = setContainerStarting(container)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Equal(t, test.status, container.Status.Get(), "metadata should not be updated")
+		} else {
+			assert.NoError(t, err)
+			assert.True(t, container.Status.Get().Starting, "starting should be set")
+			assert.NoError(t, resetContainerStarting(container))
+			assert.False(t, container.Status.Get().Starting, "starting should be reset")
+		}
+	}
+}

pkg/cri/server/container_stats.go

@@ -0,0 +1,47 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	tasks "github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ContainerStats returns stats of the container. If the container does not
+// exist, the call returns an error.
+func (c *criService) ContainerStats(ctx context.Context, in *runtime.ContainerStatsRequest) (*runtime.ContainerStatsResponse, error) {
+	cntr, err := c.containerStore.Get(in.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	request := &tasks.MetricsRequest{Filters: []string{"id==" + cntr.ID}}
+	resp, err := c.client.TaskService().Metrics(ctx, request)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch metrics for task")
+	}
+	if len(resp.Metrics) != 1 {
+		return nil, errors.Errorf("unexpected metrics response: %+v", resp.Metrics)
+	}
+
+	cs, err := c.containerMetrics(cntr.Metadata, resp.Metrics[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode container metrics")
+	}
+	return &runtime.ContainerStatsResponse{Stats: cs}, nil
+}

pkg/cri/server/container_stats_list.go

@@ -0,0 +1,116 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	tasks "github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/containerd/containerd/api/types"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// ListContainerStats returns stats of all running containers.
+func (c *criService) ListContainerStats(
+	ctx context.Context,
+	in *runtime.ListContainerStatsRequest,
+) (*runtime.ListContainerStatsResponse, error) {
+	request, containers, err := c.buildTaskMetricsRequest(in)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build metrics request")
+	}
+	resp, err := c.client.TaskService().Metrics(ctx, &request)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to fetch metrics for tasks")
+	}
+	criStats, err := c.toCRIContainerStats(resp.Metrics, containers)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to convert to cri containerd stats format")
+	}
+	return criStats, nil
+}
+
+func (c *criService) toCRIContainerStats(
+	stats []*types.Metric,
+	containers []containerstore.Container,
+) (*runtime.ListContainerStatsResponse, error) {
+	statsMap := make(map[string]*types.Metric)
+	for _, stat := range stats {
+		statsMap[stat.ID] = stat
+	}
+	containerStats := new(runtime.ListContainerStatsResponse)
+	for _, cntr := range containers {
+		cs, err := c.containerMetrics(cntr.Metadata, statsMap[cntr.ID])
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to decode container metrics for %q", cntr.ID)
+		}
+		containerStats.Stats = append(containerStats.Stats, cs)
+	}
+	return containerStats, nil
+}
+
+func (c *criService) normalizeContainerStatsFilter(filter *runtime.ContainerStatsFilter) {
+	if cntr, err := c.containerStore.Get(filter.GetId()); err == nil {
+		filter.Id = cntr.ID
+	}
+	if sb, err := c.sandboxStore.Get(filter.GetPodSandboxId()); err == nil {
+		filter.PodSandboxId = sb.ID
+	}
+}
+
+// buildTaskMetricsRequest constructs a tasks.MetricsRequest based on
+// the information in the stats request and the containerStore
+func (c *criService) buildTaskMetricsRequest(
+	r *runtime.ListContainerStatsRequest,
+) (tasks.MetricsRequest, []containerstore.Container, error) {
+	var req tasks.MetricsRequest
+	if r.GetFilter() == nil {
+		return req, nil, nil
+	}
+	c.normalizeContainerStatsFilter(r.GetFilter())
+	var containers []containerstore.Container
+	for _, cntr := range c.containerStore.List() {
+		if r.GetFilter().GetId() != "" && cntr.ID != r.GetFilter().GetId() {
+			continue
+		}
+		if r.GetFilter().GetPodSandboxId() != "" && cntr.SandboxID != r.GetFilter().GetPodSandboxId() {
+			continue
+		}
+		if r.GetFilter().GetLabelSelector() != nil &&
+			!matchLabelSelector(r.GetFilter().GetLabelSelector(), cntr.Config.GetLabels()) {
+			continue
+		}
+		containers = append(containers, cntr)
+		req.Filters = append(req.Filters, "id=="+cntr.ID)
+	}
+	return req, containers, nil
+}
+
+func matchLabelSelector(selector, labels map[string]string) bool {
+	for k, v := range selector {
+		if val, ok := labels[k]; ok {
+			if v != val {
+				return false
+			}
+		} else {
+			return false
+		}
+	}
+	return true
+}

pkg/cri/server/container_stats_list_linux.go

@@ -0,0 +1,125 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/api/types"
+	v1 "github.com/containerd/containerd/metrics/types/v1"
+	v2 "github.com/containerd/containerd/metrics/types/v2"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.snapshotStore.Get(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		s, err := typeurl.UnmarshalAny(stats.Data)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to extract container metrics")
+		}
+		switch metrics := s.(type) {
+		case *v1.Metrics:
+			if metrics.CPU != nil && metrics.CPU.Usage != nil {
+				cs.Cpu = &runtime.CpuUsage{
+					Timestamp:            stats.Timestamp.UnixNano(),
+					UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total},
+				}
+			}
+			if metrics.Memory != nil && metrics.Memory.Usage != nil {
+				cs.Memory = &runtime.MemoryUsage{
+					Timestamp: stats.Timestamp.UnixNano(),
+					WorkingSetBytes: &runtime.UInt64Value{
+						Value: getWorkingSet(metrics.Memory),
+					},
+				}
+			}
+		case *v2.Metrics:
+			if metrics.CPU != nil {
+				cs.Cpu = &runtime.CpuUsage{
+					Timestamp:            stats.Timestamp.UnixNano(),
+					UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.UsageUsec * 1000},
+				}
+			}
+			if metrics.Memory != nil {
+				cs.Memory = &runtime.MemoryUsage{
+					Timestamp: stats.Timestamp.UnixNano(),
+					WorkingSetBytes: &runtime.UInt64Value{
+						Value: getWorkingSetV2(metrics.Memory),
+					},
+				}
+			}
+		default:
+			return &cs, errors.Errorf("unxpected metrics type: %v", metrics)
+		}
+	}
+
+	return &cs, nil
+}
+
+// getWorkingSet calculates workingset memory from cgroup memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - total_inactive_file
+func getWorkingSet(memory *v1.MemoryStat) uint64 {
+	if memory.Usage == nil {
+		return 0
+	}
+	var workingSet uint64
+	if memory.TotalInactiveFile < memory.Usage.Usage {
+		workingSet = memory.Usage.Usage - memory.TotalInactiveFile
+	}
+	return workingSet
+}
+
+// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - inactive_file
+func getWorkingSetV2(memory *v2.MemoryStat) uint64 {
+	var workingSet uint64
+	if memory.InactiveFile < memory.Usage {
+		workingSet = memory.Usage - memory.InactiveFile
+	}
+	return workingSet
+}

pkg/cri/server/container_stats_list_linux_test.go

@@ -0,0 +1,55 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	v1 "github.com/containerd/cgroups/stats/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetWorkingSet(t *testing.T) {
+	for desc, test := range map[string]struct {
+		memory   *v1.MemoryStat
+		expected uint64
+	}{
+		"nil memory usage": {
+			memory:   &v1.MemoryStat{},
+			expected: 0,
+		},
+		"memory usage higher than inactive_total_file": {
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 1000,
+				Usage:             &v1.MemoryEntry{Usage: 2000},
+			},
+			expected: 1000,
+		},
+		"memory usage lower than inactive_total_file": {
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 2000,
+				Usage:             &v1.MemoryEntry{Usage: 1000},
+			},
+			expected: 0,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			got := getWorkingSet(test.memory)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}

pkg/cri/server/container_stats_list_other.go

@@ -0,0 +1,36 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	return &cs, errors.Wrap(errdefs.ErrNotImplemented, "container metrics")
+}

pkg/cri/server/container_stats_list_windows.go

@@ -0,0 +1,84 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
+	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+func (c *criService) containerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.snapshotStore.Get(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		s, err := typeurl.UnmarshalAny(stats.Data)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to extract container metrics")
+		}
+		wstats := s.(*wstats.Statistics).GetWindows()
+		if wstats == nil {
+			return nil, errors.New("windows stats is empty")
+		}
+		if wstats.Processor != nil {
+			cs.Cpu = &runtime.CpuUsage{
+				Timestamp:            wstats.Timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS},
+			}
+		}
+		if wstats.Memory != nil {
+			cs.Memory = &runtime.MemoryUsage{
+				Timestamp: wstats.Timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes,
+				},
+			}
+		}
+	}
+	return &cs, nil
+}

pkg/cri/server/container_status.go

@@ -0,0 +1,173 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// ContainerStatus inspects the container and returns the status.
+func (c *criService) ContainerStatus(ctx context.Context, r *runtime.ContainerStatusRequest) (*runtime.ContainerStatusResponse, error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	// TODO(random-liu): Clean up the following logic in CRI.
+	// Current assumption:
+	// * ImageSpec in container config is image ID.
+	// * ImageSpec in container status is image tag.
+	// * ImageRef in container status is repo digest.
+	spec := container.Config.GetImage()
+	imageRef := container.ImageRef
+	image, err := c.imageStore.Get(imageRef)
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, errors.Wrapf(err, "failed to get image %q", imageRef)
+		}
+	} else {
+		repoTags, repoDigests := parseImageReferences(image.References)
+		if len(repoTags) > 0 {
+			// Based on current behavior of dockershim, this field should be
+			// image tag.
+			spec = &runtime.ImageSpec{Image: repoTags[0]}
+		}
+		if len(repoDigests) > 0 {
+			// Based on the CRI definition, this field will be consumed by user.
+			imageRef = repoDigests[0]
+		}
+	}
+	status := toCRIContainerStatus(container, spec, imageRef)
+	if status.GetCreatedAt() == 0 {
+		// CRI doesn't allow CreatedAt == 0.
+		info, err := container.Container.Info(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get CreatedAt in %q state", status.State)
+		}
+		status.CreatedAt = info.CreatedAt.UnixNano()
+	}
+
+	info, err := toCRIContainerInfo(ctx, container, r.GetVerbose())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get verbose container info")
+	}
+
+	return &runtime.ContainerStatusResponse{
+		Status: status,
+		Info:   info,
+	}, nil
+}
+
+// toCRIContainerStatus converts internal container object to CRI container status.
+func toCRIContainerStatus(container containerstore.Container, spec *runtime.ImageSpec, imageRef string) *runtime.ContainerStatus {
+	meta := container.Metadata
+	status := container.Status.Get()
+	reason := status.Reason
+	if status.State() == runtime.ContainerState_CONTAINER_EXITED && reason == "" {
+		if status.ExitCode == 0 {
+			reason = completeExitReason
+		} else {
+			reason = errorExitReason
+		}
+	}
+
+	return &runtime.ContainerStatus{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		State:       status.State(),
+		CreatedAt:   status.CreatedAt,
+		StartedAt:   status.StartedAt,
+		FinishedAt:  status.FinishedAt,
+		ExitCode:    status.ExitCode,
+		Image:       spec,
+		ImageRef:    imageRef,
+		Reason:      reason,
+		Message:     status.Message,
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+		Mounts:      meta.Config.GetMounts(),
+		LogPath:     meta.LogPath,
+	}
+}
+
+// ContainerInfo is extra information for a container.
+type ContainerInfo struct {
+	// TODO(random-liu): Add sandboxID in CRI container status.
+	SandboxID      string                   `json:"sandboxID"`
+	Pid            uint32                   `json:"pid"`
+	Removing       bool                     `json:"removing"`
+	SnapshotKey    string                   `json:"snapshotKey"`
+	Snapshotter    string                   `json:"snapshotter"`
+	RuntimeType    string                   `json:"runtimeType"`
+	RuntimeOptions interface{}              `json:"runtimeOptions"`
+	Config         *runtime.ContainerConfig `json:"config"`
+	RuntimeSpec    *runtimespec.Spec        `json:"runtimeSpec"`
+}
+
+// toCRIContainerInfo converts internal container object information to CRI container status response info map.
+func toCRIContainerInfo(ctx context.Context, container containerstore.Container, verbose bool) (map[string]string, error) {
+	if !verbose {
+		return nil, nil
+	}
+
+	meta := container.Metadata
+	status := container.Status.Get()
+
+	// TODO(random-liu): Change CRI status info to use array instead of map.
+	ci := &ContainerInfo{
+		SandboxID: container.SandboxID,
+		Pid:       status.Pid,
+		Removing:  status.Removing,
+		Config:    meta.Config,
+	}
+
+	var err error
+	ci.RuntimeSpec, err = container.Container.Spec(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container runtime spec")
+	}
+
+	ctrInfo, err := container.Container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container info")
+	}
+	ci.SnapshotKey = ctrInfo.SnapshotKey
+	ci.Snapshotter = ctrInfo.Snapshotter
+
+	runtimeOptions, err := getRuntimeOptions(ctrInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
+	ci.RuntimeType = ctrInfo.Runtime.Name
+	ci.RuntimeOptions = runtimeOptions
+
+	infoBytes, err := json.Marshal(ci)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal info %v", ci)
+	}
+	return map[string]string{
+		"info": string(infoBytes),
+	}, nil
+}

pkg/cri/server/container_status_test.go

@@ -0,0 +1,227 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+)
+
+func getContainerStatusTestData() (*containerstore.Metadata, *containerstore.Status,
+	*imagestore.Image, *runtime.ContainerStatus) {
+	imageID := "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+	testID := "test-id"
+	config := &runtime.ContainerConfig{
+		Metadata: &runtime.ContainerMetadata{
+			Name:    "test-name",
+			Attempt: 1,
+		},
+		Image: &runtime.ImageSpec{Image: "test-image"},
+		Mounts: []*runtime.Mount{{
+			ContainerPath: "test-container-path",
+			HostPath:      "test-host-path",
+		}},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+
+	createdAt := time.Now().UnixNano()
+	startedAt := time.Now().UnixNano()
+
+	metadata := &containerstore.Metadata{
+		ID:        testID,
+		Name:      "test-long-name",
+		SandboxID: "test-sandbox-id",
+		Config:    config,
+		ImageRef:  imageID,
+		LogPath:   "test-log-path",
+	}
+	status := &containerstore.Status{
+		Pid:       1234,
+		CreatedAt: createdAt,
+		StartedAt: startedAt,
+	}
+	image := &imagestore.Image{
+		ID: imageID,
+		References: []string{
+			"gcr.io/library/busybox:latest",
+			"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+	}
+	expected := &runtime.ContainerStatus{
+		Id:          testID,
+		Metadata:    config.GetMetadata(),
+		State:       runtime.ContainerState_CONTAINER_RUNNING,
+		CreatedAt:   createdAt,
+		StartedAt:   startedAt,
+		Image:       &runtime.ImageSpec{Image: "gcr.io/library/busybox:latest"},
+		ImageRef:    "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		Reason:      completeExitReason,
+		Labels:      config.GetLabels(),
+		Annotations: config.GetAnnotations(),
+		Mounts:      config.GetMounts(),
+		LogPath:     "test-log-path",
+	}
+
+	return metadata, status, image, expected
+}
+
+func TestToCRIContainerStatus(t *testing.T) {
+	for desc, test := range map[string]struct {
+		finishedAt     int64
+		exitCode       int32
+		reason         string
+		message        string
+		expectedState  runtime.ContainerState
+		expectedReason string
+	}{
+		"container running": {
+			expectedState: runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		"container exited with reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       1,
+			reason:         "test-reason",
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: "test-reason",
+		},
+		"container exited with exit code 0 without reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       0,
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: completeExitReason,
+		},
+		"container exited with non-zero exit code without reason": {
+			finishedAt:     time.Now().UnixNano(),
+			exitCode:       1,
+			message:        "test-message",
+			expectedState:  runtime.ContainerState_CONTAINER_EXITED,
+			expectedReason: errorExitReason,
+		},
+	} {
+		metadata, status, _, expected := getContainerStatusTestData()
+		// Update status with test case.
+		status.FinishedAt = test.finishedAt
+		status.ExitCode = test.exitCode
+		status.Reason = test.reason
+		status.Message = test.message
+		container, err := containerstore.NewContainer(
+			*metadata,
+			containerstore.WithFakeStatus(*status),
+		)
+		assert.NoError(t, err)
+		// Set expectation based on test case.
+		expected.State = test.expectedState
+		expected.Reason = test.expectedReason
+		expected.FinishedAt = test.finishedAt
+		expected.ExitCode = test.exitCode
+		expected.Message = test.message
+		containerStatus := toCRIContainerStatus(container,
+			expected.Image,
+			expected.ImageRef)
+		assert.Equal(t, expected, containerStatus, desc)
+	}
+}
+
+// TODO(mikebrow): add a fake containerd container.Container.Spec client api so we can test verbose is true option
+func TestToCRIContainerInfo(t *testing.T) {
+	metadata, status, _, _ := getContainerStatusTestData()
+	container, err := containerstore.NewContainer(
+		*metadata,
+		containerstore.WithFakeStatus(*status),
+	)
+	assert.NoError(t, err)
+
+	info, err := toCRIContainerInfo(context.Background(),
+		container,
+		false)
+	assert.NoError(t, err)
+	assert.Nil(t, info)
+}
+
+func TestContainerStatus(t *testing.T) {
+	for desc, test := range map[string]struct {
+		exist         bool
+		imageExist    bool
+		finishedAt    int64
+		reason        string
+		expectedState runtime.ContainerState
+		expectErr     bool
+	}{
+		"container running": {
+			exist:         true,
+			imageExist:    true,
+			expectedState: runtime.ContainerState_CONTAINER_RUNNING,
+		},
+		"container exited": {
+			exist:         true,
+			imageExist:    true,
+			finishedAt:    time.Now().UnixNano(),
+			reason:        "test-reason",
+			expectedState: runtime.ContainerState_CONTAINER_EXITED,
+		},
+		"container not exist": {
+			exist:      false,
+			imageExist: true,
+			expectErr:  true,
+		},
+		"image not exist": {
+			exist:      false,
+			imageExist: false,
+			expectErr:  true,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		metadata, status, image, expected := getContainerStatusTestData()
+		// Update status with test case.
+		status.FinishedAt = test.finishedAt
+		status.Reason = test.reason
+		container, err := containerstore.NewContainer(
+			*metadata,
+			containerstore.WithFakeStatus(*status),
+		)
+		assert.NoError(t, err)
+		if test.exist {
+			assert.NoError(t, c.containerStore.Add(container))
+		}
+		if test.imageExist {
+			c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{*image})
+			assert.NoError(t, err)
+		}
+		resp, err := c.ContainerStatus(context.Background(), &runtime.ContainerStatusRequest{ContainerId: container.ID})
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Nil(t, resp)
+			continue
+		}
+		// Set expectation based on test case.
+		expected.FinishedAt = test.finishedAt
+		expected.Reason = test.reason
+		expected.State = test.expectedState
+		assert.Equal(t, expected, resp.GetStatus())
+	}
+}

pkg/cri/server/container_stop.go

@@ -0,0 +1,186 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd"
+	eventtypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	"github.com/containerd/containerd/pkg/cri/store"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// StopContainer stops a running container with a grace period (i.e., timeout).
+func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainerRequest) (*runtime.StopContainerResponse, error) {
+	// Get container config from container store.
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find container %q", r.GetContainerId())
+	}
+
+	if err := c.stopContainer(ctx, container, time.Duration(r.GetTimeout())*time.Second); err != nil {
+		return nil, err
+	}
+
+	return &runtime.StopContainerResponse{}, nil
+}
+
+// stopContainer stops a container based on the container metadata.
+func (c *criService) stopContainer(ctx context.Context, container containerstore.Container, timeout time.Duration) error {
+	id := container.ID
+
+	// Return without error if container is not running. This makes sure that
+	// stop only takes real action after the container is started.
+	state := container.Status.Get().State()
+	if state != runtime.ContainerState_CONTAINER_RUNNING &&
+		state != runtime.ContainerState_CONTAINER_UNKNOWN {
+		log.G(ctx).Infof("Container to stop %q must be in running or unknown state, current state %q",
+			id, criContainerStateToString(state))
+		return nil
+	}
+
+	task, err := container.Container.Task(ctx, nil)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to get task for container %q", id)
+		}
+		// Don't return for unknown state, some cleanup needs to be done.
+		if state == runtime.ContainerState_CONTAINER_UNKNOWN {
+			return cleanupUnknownContainer(ctx, id, container)
+		}
+		return nil
+	}
+
+	// Handle unknown state.
+	if state == runtime.ContainerState_CONTAINER_UNKNOWN {
+		// Start an exit handler for containers in unknown state.
+		waitCtx, waitCancel := context.WithCancel(ctrdutil.NamespacedContext())
+		defer waitCancel()
+		exitCh, err := task.Wait(waitCtx)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrapf(err, "failed to wait for task for %q", id)
+			}
+			return cleanupUnknownContainer(ctx, id, container)
+		}
+
+		exitCtx, exitCancel := context.WithCancel(context.Background())
+		stopCh := c.eventMonitor.startExitMonitor(exitCtx, id, task.Pid(), exitCh)
+		defer func() {
+			exitCancel()
+			// This ensures that exit monitor is stopped before
+			// `Wait` is cancelled, so no exit event is generated
+			// because of the `Wait` cancellation.
+			<-stopCh
+		}()
+	}
+
+	// We only need to kill the task. The event handler will Delete the
+	// task from containerd after it handles the Exited event.
+	if timeout > 0 {
+		stopSignal := "SIGTERM"
+		if container.StopSignal != "" {
+			stopSignal = container.StopSignal
+		} else {
+			// The image may have been deleted, and the `StopSignal` field is
+			// just introduced to handle that.
+			// However, for containers created before the `StopSignal` field is
+			// introduced, still try to get the stop signal from the image config.
+			// If the image has been deleted, logging an error and using the
+			// default SIGTERM is still better than returning error and leaving
+			// the container unstoppable. (See issue #990)
+			// TODO(random-liu): Remove this logic when containerd 1.2 is deprecated.
+			image, err := c.imageStore.Get(container.ImageRef)
+			if err != nil {
+				if err != store.ErrNotExist {
+					return errors.Wrapf(err, "failed to get image %q", container.ImageRef)
+				}
+				log.G(ctx).Warningf("Image %q not found, stop container with signal %q", container.ImageRef, stopSignal)
+			} else {
+				if image.ImageSpec.Config.StopSignal != "" {
+					stopSignal = image.ImageSpec.Config.StopSignal
+				}
+			}
+		}
+		sig, err := containerd.ParseSignal(stopSignal)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse stop signal %q", stopSignal)
+		}
+		log.G(ctx).Infof("Stop container %q with signal %v", id, sig)
+		if err = task.Kill(ctx, sig); err != nil && !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to stop container %q", id)
+		}
+
+		sigTermCtx, sigTermCtxCancel := context.WithTimeout(ctx, timeout)
+		defer sigTermCtxCancel()
+		err = c.waitContainerStop(sigTermCtx, container)
+		if err == nil {
+			// Container stopped on first signal no need for SIGKILL
+			return nil
+		}
+		// If the parent context was cancelled or exceeded return immediately
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// sigTermCtx was exceeded. Send SIGKILL
+		log.G(ctx).Debugf("Stop container %q with signal %v timed out", id, sig)
+	}
+
+	log.G(ctx).Infof("Kill container %q", id)
+	if err = task.Kill(ctx, syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrapf(err, "failed to kill container %q", id)
+	}
+
+	// Wait for a fixed timeout until container stop is observed by event monitor.
+	err = c.waitContainerStop(ctx, container)
+	if err != nil {
+		return errors.Wrapf(err, "an error occurs during waiting for container %q to be killed", id)
+	}
+	return nil
+}
+
+// waitContainerStop waits for container to be stopped until context is
+// cancelled or the context deadline is exceeded.
+func (c *criService) waitContainerStop(ctx context.Context, container containerstore.Container) error {
+	select {
+	case <-ctx.Done():
+		return errors.Wrapf(ctx.Err(), "wait container %q", container.ID)
+	case <-container.Stopped():
+		return nil
+	}
+}
+
+// cleanupUnknownContainer cleanup stopped container in unknown state.
+func cleanupUnknownContainer(ctx context.Context, id string, cntr containerstore.Container) error {
+	// Reuse handleContainerExit to do the cleanup.
+	return handleContainerExit(ctx, &eventtypes.TaskExit{
+		ContainerID: id,
+		ID:          id,
+		Pid:         0,
+		ExitStatus:  unknownExitCode,
+		ExitedAt:    time.Now(),
+	}, cntr)
+}

pkg/cri/server/container_stop_test.go

@@ -0,0 +1,85 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+func TestWaitContainerStop(t *testing.T) {
+	id := "test-id"
+	for desc, test := range map[string]struct {
+		status    *containerstore.Status
+		cancel    bool
+		timeout   time.Duration
+		expectErr bool
+	}{
+		"should return error if timeout exceeds": {
+			status: &containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			timeout:   200 * time.Millisecond,
+			expectErr: true,
+		},
+		"should return error if context is cancelled": {
+			status: &containerstore.Status{
+				CreatedAt: time.Now().UnixNano(),
+				StartedAt: time.Now().UnixNano(),
+			},
+			timeout:   time.Hour,
+			cancel:    true,
+			expectErr: true,
+		},
+		"should not return error if container is stopped before timeout": {
+			status: &containerstore.Status{
+				CreatedAt:  time.Now().UnixNano(),
+				StartedAt:  time.Now().UnixNano(),
+				FinishedAt: time.Now().UnixNano(),
+			},
+			timeout:   time.Hour,
+			expectErr: false,
+		},
+	} {
+		c := newTestCRIService()
+		container, err := containerstore.NewContainer(
+			containerstore.Metadata{ID: id},
+			containerstore.WithFakeStatus(*test.status),
+		)
+		assert.NoError(t, err)
+		assert.NoError(t, c.containerStore.Add(container))
+		ctx := context.Background()
+		if test.cancel {
+			cancelledCtx, cancel := context.WithCancel(ctx)
+			cancel()
+			ctx = cancelledCtx
+		}
+		if test.timeout > 0 {
+			timeoutCtx, cancel := context.WithTimeout(ctx, test.timeout)
+			defer cancel()
+			ctx = timeoutCtx
+		}
+		err = c.waitContainerStop(ctx, container)
+		assert.Equal(t, test.expectErr, err != nil, desc)
+	}
+}

pkg/cri/server/container_update_resources_linux.go

@@ -0,0 +1,148 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	gocontext "context"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/typeurl"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/containerd/opts"
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	"github.com/containerd/containerd/pkg/util"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (retRes *runtime.UpdateContainerResourcesResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	// Update resources in status update transaction, so that:
+	// 1) There won't be race condition with container start.
+	// 2) There won't be concurrent resource update to the same container.
+	if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		return status, c.updateContainerResources(ctx, container, r.GetLinux(), status)
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to update resources")
+	}
+	return &runtime.UpdateContainerResourcesResponse{}, nil
+}
+
+func (c *criService) updateContainerResources(ctx context.Context,
+	cntr containerstore.Container,
+	resources *runtime.LinuxContainerResources,
+	status containerstore.Status) (retErr error) {
+	id := cntr.ID
+	// Do not update the container when there is a removal in progress.
+	if status.Removing {
+		return errors.Errorf("container %q is in removing state", id)
+	}
+
+	// Update container spec. If the container is not started yet, updating
+	// spec makes sure that the resource limits are correct when start;
+	// if the container is already started, updating spec is still required,
+	// the spec will become our source of truth for resource limits.
+	oldSpec, err := cntr.Container.Spec(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to get container spec")
+	}
+	newSpec, err := updateOCILinuxResource(ctx, oldSpec, resources,
+		c.config.TolerateMissingHugetlbController, c.config.DisableHugetlbController)
+	if err != nil {
+		return errors.Wrap(err, "failed to update resource in spec")
+	}
+
+	if err := updateContainerSpec(ctx, cntr.Container, newSpec); err != nil {
+		return err
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// Reset spec on error.
+			if err := updateContainerSpec(deferCtx, cntr.Container, oldSpec); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to update spec %+v for container %q", oldSpec, id)
+			}
+		}
+	}()
+
+	// If container is not running, only update spec is enough, new resource
+	// limit will be applied when container start.
+	if status.State() != runtime.ContainerState_CONTAINER_RUNNING {
+		return nil
+	}
+
+	task, err := cntr.Container.Task(ctx, nil)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			// Task exited already.
+			return nil
+		}
+		return errors.Wrap(err, "failed to get task")
+	}
+	// newSpec.Linux won't be nil
+	if err := task.Update(ctx, containerd.WithResources(newSpec.Linux.Resources)); err != nil {
+		if errdefs.IsNotFound(err) {
+			// Task exited already.
+			return nil
+		}
+		return errors.Wrap(err, "failed to update resources")
+	}
+	return nil
+}
+
+// updateContainerSpec updates container spec.
+func updateContainerSpec(ctx context.Context, cntr containerd.Container, spec *runtimespec.Spec) error {
+	any, err := typeurl.MarshalAny(spec)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal spec %+v", spec)
+	}
+	if err := cntr.Update(ctx, func(ctx gocontext.Context, client *containerd.Client, c *containers.Container) error {
+		c.Spec = any
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "failed to update container spec")
+	}
+	return nil
+}
+
+// updateOCILinuxResource updates container resource limit.
+func updateOCILinuxResource(ctx context.Context, spec *runtimespec.Spec, new *runtime.LinuxContainerResources,
+	tolerateMissingHugetlbController, disableHugetlbController bool) (*runtimespec.Spec, error) {
+	// Copy to make sure old spec is not changed.
+	var cloned runtimespec.Spec
+	if err := util.DeepCopy(&cloned, spec); err != nil {
+		return nil, errors.Wrap(err, "failed to deep copy")
+	}
+	if cloned.Linux == nil {
+		cloned.Linux = &runtimespec.Linux{}
+	}
+	if err := opts.WithResources(new, tolerateMissingHugetlbController, disableHugetlbController)(ctx, nil, nil, &cloned); err != nil {
+		return nil, errors.Wrap(err, "unable to set linux container resources")
+	}
+	return &cloned, nil
+}

pkg/cri/server/container_update_resources_linux_test.go

@@ -0,0 +1,162 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestUpdateOCILinuxResource(t *testing.T) {
+	oomscoreadj := new(int)
+	*oomscoreadj = -500
+	for desc, test := range map[string]struct {
+		spec      *runtimespec.Spec
+		resources *runtime.LinuxContainerResources
+		expected  *runtimespec.Spec
+		expectErr bool
+	}{
+		"should be able to update each resource": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(1111),
+							Quota:  proto.Int64(2222),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "2-3",
+						},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuPeriod:          6666,
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetCpus:         "4-5",
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(6666),
+							Cpus:   "4-5",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+		"should skip empty fields": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(1111),
+							Quota:  proto.Int64(2222),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "2-3",
+						},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(3333),
+							Cpus:   "0-1",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+		"should be able to fill empty fields": {
+			spec: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(12345)},
+					},
+				},
+			},
+			resources: &runtime.LinuxContainerResources{
+				CpuPeriod:          6666,
+				CpuQuota:           5555,
+				CpuShares:          4444,
+				MemoryLimitInBytes: 54321,
+				OomScoreAdj:        500,
+				CpusetCpus:         "4-5",
+				CpusetMems:         "6-7",
+			},
+			expected: &runtimespec.Spec{
+				Process: &runtimespec.Process{OOMScoreAdj: oomscoreadj},
+				Linux: &runtimespec.Linux{
+					Resources: &runtimespec.LinuxResources{
+						Memory: &runtimespec.LinuxMemory{Limit: proto.Int64(54321)},
+						CPU: &runtimespec.LinuxCPU{
+							Shares: proto.Uint64(4444),
+							Quota:  proto.Int64(5555),
+							Period: proto.Uint64(6666),
+							Cpus:   "4-5",
+							Mems:   "6-7",
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got, err := updateOCILinuxResource(context.Background(), test.spec, test.resources, false, false)
+		if test.expectErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+		assert.Equal(t, test.expected, got)
+	}
+}

pkg/cri/server/container_update_resources_other.go

@@ -0,0 +1,44 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (retRes *runtime.UpdateContainerResourcesResponse, retErr error) {
+	container, err := c.containerStore.Get(r.GetContainerId())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find container")
+	}
+	// Update resources in status update transaction, so that:
+	// 1) There won't be race condition with container start.
+	// 2) There won't be concurrent resource update to the same container.
+	if err := container.Status.Update(func(status containerstore.Status) (containerstore.Status, error) {
+		return status, nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to update resources")
+	}
+	return &runtime.UpdateContainerResourcesResponse{}, nil
+}

pkg/cri/server/container_update_resources_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/errdefs"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// UpdateContainerResources updates ContainerConfig of the container.
+// TODO(windows): Figure out whether windows support this.
+func (c *criService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (*runtime.UpdateContainerResourcesResponse, error) {
+	return nil, errdefs.ErrNotImplemented
+}

pkg/cri/server/events.go

@@ -0,0 +1,461 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd"
+	eventtypes "github.com/containerd/containerd/api/events"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/typeurl"
+	gogotypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/util/clock"
+
+	"github.com/containerd/containerd/pkg/constants"
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	"github.com/containerd/containerd/pkg/cri/store"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+const (
+	backOffInitDuration        = 1 * time.Second
+	backOffMaxDuration         = 5 * time.Minute
+	backOffExpireCheckDuration = 1 * time.Second
+
+	// handleEventTimeout is the timeout for handling 1 event. Event monitor
+	// handles events in serial, if one event blocks the event monitor, no
+	// other events can be handled.
+	// Add a timeout for each event handling, events that timeout will be requeued and
+	// handled again in the future.
+	handleEventTimeout = 10 * time.Second
+
+	exitChannelSize = 1024
+)
+
+// eventMonitor monitors containerd event and updates internal state correspondingly.
+// TODO(random-liu): Handle event for each container in a separate goroutine.
+type eventMonitor struct {
+	c  *criService
+	ch <-chan *events.Envelope
+	// exitCh receives container/sandbox exit events from exit monitors.
+	exitCh  chan *eventtypes.TaskExit
+	errCh   <-chan error
+	ctx     context.Context
+	cancel  context.CancelFunc
+	backOff *backOff
+}
+
+type backOff struct {
+	queuePool map[string]*backOffQueue
+	// tickerMu is mutex used to protect the ticker.
+	tickerMu      sync.Mutex
+	ticker        *time.Ticker
+	minDuration   time.Duration
+	maxDuration   time.Duration
+	checkDuration time.Duration
+	clock         clock.Clock
+}
+
+type backOffQueue struct {
+	events     []interface{}
+	expireTime time.Time
+	duration   time.Duration
+	clock      clock.Clock
+}
+
+// Create new event monitor. New event monitor will start subscribing containerd event. All events
+// happen after it should be monitored.
+func newEventMonitor(c *criService) *eventMonitor {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &eventMonitor{
+		c:       c,
+		ctx:     ctx,
+		cancel:  cancel,
+		exitCh:  make(chan *eventtypes.TaskExit, exitChannelSize),
+		backOff: newBackOff(),
+	}
+}
+
+// subscribe starts to subscribe containerd events.
+func (em *eventMonitor) subscribe(subscriber events.Subscriber) {
+	// note: filters are any match, if you want any match but not in namespace foo
+	// then you have to manually filter namespace foo
+	filters := []string{
+		`topic=="/tasks/oom"`,
+		`topic~="/images/"`,
+	}
+	em.ch, em.errCh = subscriber.Subscribe(em.ctx, filters...)
+}
+
+// startExitMonitor starts an exit monitor for a given container/sandbox.
+func (em *eventMonitor) startExitMonitor(ctx context.Context, id string, pid uint32, exitCh <-chan containerd.ExitStatus) <-chan struct{} {
+	stopCh := make(chan struct{})
+	go func() {
+		defer close(stopCh)
+		select {
+		case exitRes := <-exitCh:
+			exitStatus, exitedAt, err := exitRes.Result()
+			if err != nil {
+				logrus.WithError(err).Errorf("Failed to get task exit status for %q", id)
+				exitStatus = unknownExitCode
+				exitedAt = time.Now()
+			}
+			em.exitCh <- &eventtypes.TaskExit{
+				ContainerID: id,
+				ID:          id,
+				Pid:         pid,
+				ExitStatus:  exitStatus,
+				ExitedAt:    exitedAt,
+			}
+		case <-ctx.Done():
+		}
+	}()
+	return stopCh
+}
+
+func convertEvent(e *gogotypes.Any) (string, interface{}, error) {
+	id := ""
+	evt, err := typeurl.UnmarshalAny(e)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to unmarshalany")
+	}
+
+	switch e := evt.(type) {
+	case *eventtypes.TaskOOM:
+		id = e.ContainerID
+	case *eventtypes.ImageCreate:
+		id = e.Name
+	case *eventtypes.ImageUpdate:
+		id = e.Name
+	case *eventtypes.ImageDelete:
+		id = e.Name
+	default:
+		return "", nil, errors.New("unsupported event")
+	}
+	return id, evt, nil
+}
+
+// start starts the event monitor which monitors and handles all subscribed events. It returns
+// an error channel for the caller to wait for stop errors from the event monitor.
+// start must be called after subscribe.
+func (em *eventMonitor) start() <-chan error {
+	errCh := make(chan error)
+	if em.ch == nil || em.errCh == nil {
+		panic("event channel is nil")
+	}
+	backOffCheckCh := em.backOff.start()
+	go func() {
+		defer close(errCh)
+		for {
+			select {
+			case e := <-em.exitCh:
+				logrus.Debugf("Received exit event %+v", e)
+				id := e.ID
+				if em.backOff.isInBackOff(id) {
+					logrus.Infof("Events for %q is in backoff, enqueue event %+v", id, e)
+					em.backOff.enBackOff(id, e)
+					break
+				}
+				if err := em.handleEvent(e); err != nil {
+					logrus.WithError(err).Errorf("Failed to handle exit event %+v for %s", e, id)
+					em.backOff.enBackOff(id, e)
+				}
+			case e := <-em.ch:
+				logrus.Debugf("Received containerd event timestamp - %v, namespace - %q, topic - %q", e.Timestamp, e.Namespace, e.Topic)
+				if e.Namespace != constants.K8sContainerdNamespace {
+					logrus.Debugf("Ignoring events in namespace - %q", e.Namespace)
+					break
+				}
+				id, evt, err := convertEvent(e.Event)
+				if err != nil {
+					logrus.WithError(err).Errorf("Failed to convert event %+v", e)
+					break
+				}
+				if em.backOff.isInBackOff(id) {
+					logrus.Infof("Events for %q is in backoff, enqueue event %+v", id, evt)
+					em.backOff.enBackOff(id, evt)
+					break
+				}
+				if err := em.handleEvent(evt); err != nil {
+					logrus.WithError(err).Errorf("Failed to handle event %+v for %s", evt, id)
+					em.backOff.enBackOff(id, evt)
+				}
+			case err := <-em.errCh:
+				// Close errCh in defer directly if there is no error.
+				if err != nil {
+					logrus.WithError(err).Errorf("Failed to handle event stream")
+					errCh <- err
+				}
+				return
+			case <-backOffCheckCh:
+				ids := em.backOff.getExpiredIDs()
+				for _, id := range ids {
+					queue := em.backOff.deBackOff(id)
+					for i, any := range queue.events {
+						if err := em.handleEvent(any); err != nil {
+							logrus.WithError(err).Errorf("Failed to handle backOff event %+v for %s", any, id)
+							em.backOff.reBackOff(id, queue.events[i:], queue.duration)
+							break
+						}
+					}
+				}
+			}
+		}
+	}()
+	return errCh
+}
+
+// stop stops the event monitor. It will close the event channel.
+// Once event monitor is stopped, it can't be started.
+func (em *eventMonitor) stop() {
+	em.backOff.stop()
+	em.cancel()
+}
+
+// handleEvent handles a containerd event.
+func (em *eventMonitor) handleEvent(any interface{}) error {
+	ctx := ctrdutil.NamespacedContext()
+	ctx, cancel := context.WithTimeout(ctx, handleEventTimeout)
+	defer cancel()
+
+	switch e := any.(type) {
+	case *eventtypes.TaskExit:
+		logrus.Infof("TaskExit event %+v", e)
+		// Use ID instead of ContainerID to rule out TaskExit event for exec.
+		cntr, err := em.c.containerStore.Get(e.ID)
+		if err == nil {
+			if err := handleContainerExit(ctx, e, cntr); err != nil {
+				return errors.Wrap(err, "failed to handle container TaskExit event")
+			}
+			return nil
+		} else if err != store.ErrNotExist {
+			return errors.Wrap(err, "can't find container for TaskExit event")
+		}
+		sb, err := em.c.sandboxStore.Get(e.ID)
+		if err == nil {
+			if err := handleSandboxExit(ctx, e, sb); err != nil {
+				return errors.Wrap(err, "failed to handle sandbox TaskExit event")
+			}
+			return nil
+		} else if err != store.ErrNotExist {
+			return errors.Wrap(err, "can't find sandbox for TaskExit event")
+		}
+		return nil
+	case *eventtypes.TaskOOM:
+		logrus.Infof("TaskOOM event %+v", e)
+		// For TaskOOM, we only care which container it belongs to.
+		cntr, err := em.c.containerStore.Get(e.ContainerID)
+		if err != nil {
+			if err != store.ErrNotExist {
+				return errors.Wrap(err, "can't find container for TaskOOM event")
+			}
+			return nil
+		}
+		err = cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+			status.Reason = oomExitReason
+			return status, nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "failed to update container status for TaskOOM event")
+		}
+	case *eventtypes.ImageCreate:
+		logrus.Infof("ImageCreate event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	case *eventtypes.ImageUpdate:
+		logrus.Infof("ImageUpdate event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	case *eventtypes.ImageDelete:
+		logrus.Infof("ImageDelete event %+v", e)
+		return em.c.updateImage(ctx, e.Name)
+	}
+
+	return nil
+}
+
+// handleContainerExit handles TaskExit event for container.
+func handleContainerExit(ctx context.Context, e *eventtypes.TaskExit, cntr containerstore.Container) error {
+	// Attach container IO so that `Delete` could cleanup the stream properly.
+	task, err := cntr.Container.Task(ctx,
+		func(*containerdio.FIFOSet) (containerdio.IO, error) {
+			// We can't directly return cntr.IO here, because
+			// even if cntr.IO is nil, the cio.IO interface
+			// is not.
+			// See https://tour.golang.org/methods/12:
+			//   Note that an interface value that holds a nil
+			//   concrete value is itself non-nil.
+			if cntr.IO != nil {
+				return cntr.IO, nil
+			}
+			return nil, nil
+		},
+	)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrapf(err, "failed to load task for container")
+		}
+	} else {
+		// TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker
+		if _, err = task.Delete(ctx, WithNRISandboxDelete(cntr.SandboxID), containerd.WithProcessKill); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrap(err, "failed to stop container")
+			}
+			// Move on to make sure container status is updated.
+		}
+	}
+	err = cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) {
+		// If FinishedAt has been set (e.g. with start failure), keep as
+		// it is.
+		if status.FinishedAt != 0 {
+			return status, nil
+		}
+		status.Pid = 0
+		status.FinishedAt = e.ExitedAt.UnixNano()
+		status.ExitCode = int32(e.ExitStatus)
+		// Unknown state can only transit to EXITED state, so we need
+		// to handle unknown state here.
+		if status.Unknown {
+			logrus.Debugf("Container %q transited from UNKNOWN to EXITED", cntr.ID)
+			status.Unknown = false
+		}
+		return status, nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to update container state")
+	}
+	// Using channel to propagate the information of container stop
+	cntr.Stop()
+	return nil
+}
+
+// handleSandboxExit handles TaskExit event for sandbox.
+func handleSandboxExit(ctx context.Context, e *eventtypes.TaskExit, sb sandboxstore.Sandbox) error {
+	// No stream attached to sandbox container.
+	task, err := sb.Container.Task(ctx, nil)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrap(err, "failed to load task for sandbox")
+		}
+	} else {
+		// TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker
+		if _, err = task.Delete(ctx, WithNRISandboxDelete(sb.ID), containerd.WithProcessKill); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrap(err, "failed to stop sandbox")
+			}
+			// Move on to make sure container status is updated.
+		}
+	}
+	err = sb.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
+		status.State = sandboxstore.StateNotReady
+		status.Pid = 0
+		return status, nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to update sandbox state")
+	}
+	// Using channel to propagate the information of sandbox stop
+	sb.Stop()
+	return nil
+}
+
+func newBackOff() *backOff {
+	return &backOff{
+		queuePool:     map[string]*backOffQueue{},
+		minDuration:   backOffInitDuration,
+		maxDuration:   backOffMaxDuration,
+		checkDuration: backOffExpireCheckDuration,
+		clock:         clock.RealClock{},
+	}
+}
+
+func (b *backOff) getExpiredIDs() []string {
+	var ids []string
+	for id, q := range b.queuePool {
+		if q.isExpire() {
+			ids = append(ids, id)
+		}
+	}
+	return ids
+}
+
+func (b *backOff) isInBackOff(key string) bool {
+	if _, ok := b.queuePool[key]; ok {
+		return true
+	}
+	return false
+}
+
+// enBackOff start to backOff and put event to the tail of queue
+func (b *backOff) enBackOff(key string, evt interface{}) {
+	if queue, ok := b.queuePool[key]; ok {
+		queue.events = append(queue.events, evt)
+		return
+	}
+	b.queuePool[key] = newBackOffQueue([]interface{}{evt}, b.minDuration, b.clock)
+}
+
+// enBackOff get out the whole queue
+func (b *backOff) deBackOff(key string) *backOffQueue {
+	queue := b.queuePool[key]
+	delete(b.queuePool, key)
+	return queue
+}
+
+// enBackOff start to backOff again and put events to the queue
+func (b *backOff) reBackOff(key string, events []interface{}, oldDuration time.Duration) {
+	duration := 2 * oldDuration
+	if duration > b.maxDuration {
+		duration = b.maxDuration
+	}
+	b.queuePool[key] = newBackOffQueue(events, duration, b.clock)
+}
+
+func (b *backOff) start() <-chan time.Time {
+	b.tickerMu.Lock()
+	defer b.tickerMu.Unlock()
+	b.ticker = time.NewTicker(b.checkDuration)
+	return b.ticker.C
+}
+
+func (b *backOff) stop() {
+	b.tickerMu.Lock()
+	defer b.tickerMu.Unlock()
+	if b.ticker != nil {
+		b.ticker.Stop()
+	}
+}
+
+func newBackOffQueue(events []interface{}, init time.Duration, c clock.Clock) *backOffQueue {
+	return &backOffQueue{
+		events:     events,
+		duration:   init,
+		expireTime: c.Now().Add(init),
+		clock:      c,
+	}
+}
+
+func (q *backOffQueue) isExpire() bool {
+	// return time.Now >= expireTime
+	return !q.clock.Now().Before(q.expireTime)
+}

pkg/cri/server/events_test.go

@@ -0,0 +1,134 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	eventtypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/typeurl"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/clock"
+)
+
+// TestBackOff tests the logic of backOff struct.
+func TestBackOff(t *testing.T) {
+	testStartTime := time.Now()
+	testClock := clock.NewFakeClock(testStartTime)
+	inputQueues := map[string]*backOffQueue{
+		"container1": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+			},
+		},
+		"container2": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+			},
+		},
+	}
+	expectedQueues := map[string]*backOffQueue{
+		"container2": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+				&eventtypes.TaskOOM{ContainerID: "container2"},
+			},
+			expireTime: testClock.Now().Add(backOffInitDuration),
+			duration:   backOffInitDuration,
+			clock:      testClock,
+		},
+		"container1": {
+			events: []interface{}{
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+				&eventtypes.TaskOOM{ContainerID: "container1"},
+			},
+			expireTime: testClock.Now().Add(backOffInitDuration),
+			duration:   backOffInitDuration,
+			clock:      testClock,
+		},
+	}
+
+	t.Logf("Should be able to backOff a event")
+	actual := newBackOff()
+	actual.clock = testClock
+	for k, queue := range inputQueues {
+		for _, event := range queue.events {
+			actual.enBackOff(k, event)
+		}
+	}
+	assert.Equal(t, actual.queuePool, expectedQueues)
+
+	t.Logf("Should be able to check if the container is in backOff state")
+	for k, queue := range inputQueues {
+		for _, e := range queue.events {
+			any, err := typeurl.MarshalAny(e)
+			assert.NoError(t, err)
+			key, _, err := convertEvent(any)
+			assert.NoError(t, err)
+			assert.Equal(t, k, key)
+			assert.Equal(t, actual.isInBackOff(key), true)
+		}
+	}
+
+	t.Logf("Should be able to check that a container isn't in backOff state")
+	notExistKey := "containerNotExist"
+	assert.Equal(t, actual.isInBackOff(notExistKey), false)
+
+	t.Logf("No containers should be expired")
+	assert.Empty(t, actual.getExpiredIDs())
+
+	t.Logf("Should be able to get all keys which are expired for backOff")
+	testClock.Sleep(backOffInitDuration)
+	actKeyList := actual.getExpiredIDs()
+	assert.Equal(t, len(inputQueues), len(actKeyList))
+	for k := range inputQueues {
+		assert.Contains(t, actKeyList, k)
+	}
+
+	t.Logf("Should be able to get out all backOff events")
+	doneQueues := map[string]*backOffQueue{}
+	for k := range inputQueues {
+		actQueue := actual.deBackOff(k)
+		doneQueues[k] = actQueue
+		assert.Equal(t, actQueue, expectedQueues[k])
+	}
+
+	t.Logf("Should not get out the event again after having got out the backOff event")
+	for k := range inputQueues {
+		var expect *backOffQueue
+		actQueue := actual.deBackOff(k)
+		assert.Equal(t, actQueue, expect)
+	}
+
+	t.Logf("Should be able to reBackOff")
+	for k, queue := range doneQueues {
+		failEventIndex := 1
+		events := queue.events[failEventIndex:]
+		actual.reBackOff(k, events, queue.duration)
+		actQueue := actual.deBackOff(k)
+		expQueue := &backOffQueue{
+			events:     events,
+			expireTime: testClock.Now().Add(2 * queue.duration),
+			duration:   2 * queue.duration,
+			clock:      testClock,
+		}
+		assert.Equal(t, actQueue, expQueue)
+	}
+}

pkg/cri/server/helpers.go

@@ -0,0 +1,390 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/typeurl"
+	imagedigest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	runtimeoptions "github.com/containerd/containerd/pkg/cri/runtimeoptions/v1"
+	"github.com/containerd/containerd/pkg/cri/store"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+const (
+	// errorStartReason is the exit reason when fails to start container.
+	errorStartReason = "StartError"
+	// errorStartExitCode is the exit code when fails to start container.
+	// 128 is the same with Docker's behavior.
+	// TODO(windows): Figure out what should be used for windows.
+	errorStartExitCode = 128
+	// completeExitReason is the exit reason when container exits with code 0.
+	completeExitReason = "Completed"
+	// errorExitReason is the exit reason when container exits with code non-zero.
+	errorExitReason = "Error"
+	// oomExitReason is the exit reason when process in container is oom killed.
+	oomExitReason = "OOMKilled"
+
+	// sandboxesDir contains all sandbox root. A sandbox root is the running
+	// directory of the sandbox, all files created for the sandbox will be
+	// placed under this directory.
+	sandboxesDir = "sandboxes"
+	// containersDir contains all container root.
+	containersDir = "containers"
+	// Delimiter used to construct container/sandbox names.
+	nameDelimiter = "_"
+
+	// criContainerdPrefix is common prefix for cri-containerd
+	criContainerdPrefix = "io.cri-containerd"
+	// containerKindLabel is a label key indicating container is sandbox container or application container
+	containerKindLabel = criContainerdPrefix + ".kind"
+	// containerKindSandbox is a label value indicating container is sandbox container
+	containerKindSandbox = "sandbox"
+	// containerKindContainer is a label value indicating container is application container
+	containerKindContainer = "container"
+	// imageLabelKey is the label key indicating the image is managed by cri plugin.
+	imageLabelKey = criContainerdPrefix + ".image"
+	// imageLabelValue is the label value indicating the image is managed by cri plugin.
+	imageLabelValue = "managed"
+	// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
+	sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
+	// containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest
+	containerMetadataExtension = criContainerdPrefix + ".container.metadata"
+
+	// defaultIfName is the default network interface for the pods
+	defaultIfName = "eth0"
+
+	// runtimeRunhcsV1 is the runtime type for runhcs.
+	runtimeRunhcsV1 = "io.containerd.runhcs.v1"
+)
+
+// makeSandboxName generates sandbox name from sandbox metadata. The name
+// generated is unique as long as sandbox metadata is unique.
+func makeSandboxName(s *runtime.PodSandboxMetadata) string {
+	return strings.Join([]string{
+		s.Name,                       // 0
+		s.Namespace,                  // 1
+		s.Uid,                        // 2
+		fmt.Sprintf("%d", s.Attempt), // 3
+	}, nameDelimiter)
+}
+
+// makeContainerName generates container name from sandbox and container metadata.
+// The name generated is unique as long as the sandbox container combination is
+// unique.
+func makeContainerName(c *runtime.ContainerMetadata, s *runtime.PodSandboxMetadata) string {
+	return strings.Join([]string{
+		c.Name,                       // 0
+		s.Name,                       // 1: pod name
+		s.Namespace,                  // 2: pod namespace
+		s.Uid,                        // 3: pod uid
+		fmt.Sprintf("%d", c.Attempt), // 4
+	}, nameDelimiter)
+}
+
+// getSandboxRootDir returns the root directory for managing sandbox files,
+// e.g. hosts files.
+func (c *criService) getSandboxRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, sandboxesDir, id)
+}
+
+// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
+// e.g. named pipes.
+func (c *criService) getVolatileSandboxRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, sandboxesDir, id)
+}
+
+// getContainerRootDir returns the root directory for managing container files,
+// e.g. state checkpoint.
+func (c *criService) getContainerRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, containersDir, id)
+}
+
+// getVolatileContainerRootDir returns the root directory for managing volatile container files,
+// e.g. named pipes.
+func (c *criService) getVolatileContainerRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, containersDir, id)
+}
+
+// criContainerStateToString formats CRI container state to string.
+func criContainerStateToString(state runtime.ContainerState) string {
+	return runtime.ContainerState_name[int32(state)]
+}
+
+// getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference.
+func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) {
+	var repoTag, repoDigest string
+	if _, ok := namedRef.(docker.NamedTagged); ok {
+		repoTag = namedRef.String()
+	}
+	if _, ok := namedRef.(docker.Canonical); ok {
+		repoDigest = namedRef.String()
+	} else if !schema1 {
+		// digest is not actual repo digest for schema1 image.
+		repoDigest = namedRef.Name() + "@" + digest.String()
+	}
+	return repoDigest, repoTag
+}
+
+// localResolve resolves image reference locally and returns corresponding image metadata. It
+// returns store.ErrNotExist if the reference doesn't exist.
+func (c *criService) localResolve(refOrID string) (imagestore.Image, error) {
+	getImageID := func(refOrId string) string {
+		if _, err := imagedigest.Parse(refOrID); err == nil {
+			return refOrID
+		}
+		return func(ref string) string {
+			// ref is not image id, try to resolve it locally.
+			// TODO(random-liu): Handle this error better for debugging.
+			normalized, err := docker.ParseDockerRef(ref)
+			if err != nil {
+				return ""
+			}
+			id, err := c.imageStore.Resolve(normalized.String())
+			if err != nil {
+				return ""
+			}
+			return id
+		}(refOrID)
+	}
+
+	imageID := getImageID(refOrID)
+	if imageID == "" {
+		// Try to treat ref as imageID
+		imageID = refOrID
+	}
+	return c.imageStore.Get(imageID)
+}
+
+// toContainerdImage converts an image object in image store to containerd image handler.
+func (c *criService) toContainerdImage(ctx context.Context, image imagestore.Image) (containerd.Image, error) {
+	// image should always have at least one reference.
+	if len(image.References) == 0 {
+		return nil, errors.Errorf("invalid image with no reference %q", image.ID)
+	}
+	return c.client.GetImage(ctx, image.References[0])
+}
+
+// getUserFromImage gets uid or user name of the image user.
+// If user is numeric, it will be treated as uid; or else, it is treated as user name.
+func getUserFromImage(user string) (*int64, string) {
+	// return both empty if user is not specified in the image.
+	if user == "" {
+		return nil, ""
+	}
+	// split instances where the id may contain user:group
+	user = strings.Split(user, ":")[0]
+	// user could be either uid or user name. Try to interpret as numeric uid.
+	uid, err := strconv.ParseInt(user, 10, 64)
+	if err != nil {
+		// If user is non numeric, assume it's user name.
+		return nil, user
+	}
+	// If user is a numeric uid.
+	return &uid, ""
+}
+
+// ensureImageExists returns corresponding metadata of the image reference, if image is not
+// pulled yet, the function will pull the image.
+func (c *criService) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig) (*imagestore.Image, error) {
+	image, err := c.localResolve(ref)
+	if err != nil && err != store.ErrNotExist {
+		return nil, errors.Wrapf(err, "failed to get image %q", ref)
+	}
+	if err == nil {
+		return &image, nil
+	}
+	// Pull image to ensure the image exists
+	resp, err := c.PullImage(ctx, &runtime.PullImageRequest{Image: &runtime.ImageSpec{Image: ref}, SandboxConfig: config})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to pull image %q", ref)
+	}
+	imageID := resp.GetImageRef()
+	newImage, err := c.imageStore.Get(imageID)
+	if err != nil {
+		// It's still possible that someone removed the image right after it is pulled.
+		return nil, errors.Wrapf(err, "failed to get image %q after pulling", imageID)
+	}
+	return &newImage, nil
+}
+
+// isInCRIMounts checks whether a destination is in CRI mount list.
+func isInCRIMounts(dst string, mounts []*runtime.Mount) bool {
+	for _, m := range mounts {
+		if filepath.Clean(m.ContainerPath) == filepath.Clean(dst) {
+			return true
+		}
+	}
+	return false
+}
+
+// filterLabel returns a label filter. Use `%q` here because containerd
+// filter needs extra quote to work properly.
+func filterLabel(k, v string) string {
+	return fmt.Sprintf("labels.%q==%q", k, v)
+}
+
+// buildLabel builds the labels from config to be passed to containerd
+func buildLabels(configLabels map[string]string, containerType string) map[string]string {
+	labels := make(map[string]string)
+	for k, v := range configLabels {
+		labels[k] = v
+	}
+	labels[containerKindLabel] = containerType
+	return labels
+}
+
+// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
+func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
+	return &runtime.AuthConfig{
+		Username:      a.Username,
+		Password:      a.Password,
+		Auth:          a.Auth,
+		IdentityToken: a.IdentityToken,
+	}
+}
+
+// parseImageReferences parses a list of arbitrary image references and returns
+// the repotags and repodigests
+func parseImageReferences(refs []string) ([]string, []string) {
+	var tags, digests []string
+	for _, ref := range refs {
+		parsed, err := docker.ParseAnyReference(ref)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsed.(docker.Canonical); ok {
+			digests = append(digests, parsed.String())
+		} else if _, ok := parsed.(docker.Tagged); ok {
+			tags = append(tags, parsed.String())
+		}
+	}
+	return tags, digests
+}
+
+// generateRuntimeOptions generates runtime options from cri plugin config.
+func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
+	if r.Options == nil {
+		if r.Type != plugin.RuntimeLinuxV1 {
+			return nil, nil
+		}
+		// This is a legacy config, generate runctypes.RuncOptions.
+		return &runctypes.RuncOptions{
+			Runtime:       r.Engine,
+			RuntimeRoot:   r.Root,
+			SystemdCgroup: c.SystemdCgroup,
+		}, nil
+	}
+	options := getRuntimeOptionsType(r.Type)
+	if err := toml.PrimitiveDecode(*r.Options, options); err != nil {
+		return nil, err
+	}
+	return options, nil
+}
+
+// getRuntimeOptionsType gets empty runtime options by the runtime type name.
+func getRuntimeOptionsType(t string) interface{} {
+	switch t {
+	case plugin.RuntimeRuncV1:
+		fallthrough
+	case plugin.RuntimeRuncV2:
+		return &runcoptions.Options{}
+	case plugin.RuntimeLinuxV1:
+		return &runctypes.RuncOptions{}
+	case runtimeRunhcsV1:
+		return &runhcsoptions.Options{}
+	default:
+		return &runtimeoptions.Options{}
+	}
+}
+
+// getRuntimeOptions get runtime options from container metadata.
+func getRuntimeOptions(c containers.Container) (interface{}, error) {
+	if c.Runtime.Options == nil {
+		return nil, nil
+	}
+	opts, err := typeurl.UnmarshalAny(c.Runtime.Options)
+	if err != nil {
+		return nil, err
+	}
+	return opts, nil
+}
+
+const (
+	// unknownExitCode is the exit code when exit reason is unknown.
+	unknownExitCode = 255
+	// unknownExitReason is the exit reason when exit reason is unknown.
+	unknownExitReason = "Unknown"
+)
+
+// unknownContainerStatus returns the default container status when its status is unknown.
+func unknownContainerStatus() containerstore.Status {
+	return containerstore.Status{
+		CreatedAt:  0,
+		StartedAt:  0,
+		FinishedAt: 0,
+		ExitCode:   unknownExitCode,
+		Reason:     unknownExitReason,
+		Unknown:    true,
+	}
+}
+
+// unknownSandboxStatus returns the default sandbox status when its status is unknown.
+func unknownSandboxStatus() sandboxstore.Status {
+	return sandboxstore.Status{
+		State: sandboxstore.StateUnknown,
+	}
+}
+
+// getPassthroughAnnotations filters requested pod annotations by comparing
+// against permitted annotations for the given runtime.
+func getPassthroughAnnotations(podAnnotations map[string]string,
+	runtimePodAnnotations []string) (passthroughAnnotations map[string]string) {
+	passthroughAnnotations = make(map[string]string)
+
+	for podAnnotationKey, podAnnotationValue := range podAnnotations {
+		for _, pattern := range runtimePodAnnotations {
+			// Use path.Match instead of filepath.Match here.
+			// filepath.Match treated `\\` as path separator
+			// on windows, which is not what we want.
+			if ok, _ := path.Match(pattern, podAnnotationKey); ok {
+				passthroughAnnotations[podAnnotationKey] = podAnnotationValue
+			}
+		}
+	}
+	return passthroughAnnotations
+}

pkg/cri/server/helpers_linux.go

@@ -0,0 +1,290 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/pkg/seccomp"
+	"github.com/containerd/containerd/pkg/seutil"
+	runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+const (
+	// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
+	defaultSandboxOOMAdj = -998
+	// defaultShmSize is the default size of the sandbox shm.
+	defaultShmSize = int64(1024 * 1024 * 64)
+	// relativeRootfsPath is the rootfs path relative to bundle path.
+	relativeRootfsPath = "rootfs"
+	// According to http://man7.org/linux/man-pages/man5/resolv.conf.5.html:
+	// "The search list is currently limited to six domains with a total of 256 characters."
+	maxDNSSearches = 6
+	// devShm is the default path of /dev/shm.
+	devShm = "/dev/shm"
+	// etcHosts is the default path of /etc/hosts file.
+	etcHosts = "/etc/hosts"
+	// etcHostname is the default path of /etc/hostname file.
+	etcHostname = "/etc/hostname"
+	// resolvConfPath is the abs path of resolv.conf on host or container.
+	resolvConfPath = "/etc/resolv.conf"
+	// hostnameEnv is the key for HOSTNAME env.
+	hostnameEnv = "HOSTNAME"
+)
+
+// getCgroupsPath generates container cgroups path.
+func getCgroupsPath(cgroupsParent, id string) string {
+	base := path.Base(cgroupsParent)
+	if strings.HasSuffix(base, ".slice") {
+		// For a.slice/b.slice/c.slice, base is c.slice.
+		// runc systemd cgroup path format is "slice:prefix:name".
+		return strings.Join([]string{base, "cri-containerd", id}, ":")
+	}
+	return filepath.Join(cgroupsParent, id)
+}
+
+// getSandboxHostname returns the hostname file path inside the sandbox root directory.
+func (c *criService) getSandboxHostname(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hostname")
+}
+
+// getSandboxHosts returns the hosts file path inside the sandbox root directory.
+func (c *criService) getSandboxHosts(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hosts")
+}
+
+// getResolvPath returns resolv.conf filepath for specified sandbox.
+func (c *criService) getResolvPath(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
+}
+
+// getSandboxDevShm returns the shm file path inside the sandbox root directory.
+func (c *criService) getSandboxDevShm(id string) string {
+	return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
+}
+
+func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
+	var labels []string
+
+	if selinuxOptions == nil {
+		return nil, nil
+	}
+	if err := checkSelinuxLevel(selinuxOptions.Level); err != nil {
+		return nil, err
+	}
+	if selinuxOptions.User != "" {
+		labels = append(labels, "user:"+selinuxOptions.User)
+	}
+	if selinuxOptions.Role != "" {
+		labels = append(labels, "role:"+selinuxOptions.Role)
+	}
+	if selinuxOptions.Type != "" {
+		labels = append(labels, "type:"+selinuxOptions.Type)
+	}
+	if selinuxOptions.Level != "" {
+		labels = append(labels, "level:"+selinuxOptions.Level)
+	}
+
+	return labels, nil
+}
+
+func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
+	labels, err := toLabel(selinuxOpts)
+	if err != nil {
+		return "", "", err
+	}
+	return label.InitLabels(labels)
+}
+
+func checkSelinuxLevel(level string) error {
+	if len(level) == 0 {
+		return nil
+	}
+
+	matched, err := regexp.MatchString(`^s\d(-s\d)??(:c\d{1,4}(\.c\d{1,4})?(,c\d{1,4}(\.c\d{1,4})?)*)?$`, level)
+	if err != nil {
+		return errors.Wrapf(err, "the format of 'level' %q is not correct", level)
+	}
+	if !matched {
+		return fmt.Errorf("the format of 'level' %q is not correct", level)
+	}
+	return nil
+}
+
+func (c *criService) apparmorEnabled() bool {
+	return runcapparmor.IsEnabled() && !c.config.DisableApparmor
+}
+
+func (c *criService) seccompEnabled() bool {
+	return seccomp.IsEnabled()
+}
+
+// openLogFile opens/creates a container log file.
+func openLogFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+// unmountRecursive unmounts the target and all mounts underneath, starting with
+// the deepest mount first.
+func unmountRecursive(ctx context.Context, target string) error {
+	mounts, err := mount.Self()
+	if err != nil {
+		return err
+	}
+
+	var toUnmount []string
+	for _, m := range mounts {
+		p, err := filepath.Rel(target, m.Mountpoint)
+		if err != nil {
+			return err
+		}
+		if !strings.HasPrefix(p, "..") {
+			toUnmount = append(toUnmount, m.Mountpoint)
+		}
+	}
+
+	// Make the deepest mount be first
+	sort.Slice(toUnmount, func(i, j int) bool {
+		return len(toUnmount[i]) > len(toUnmount[j])
+	})
+
+	for i, mountPath := range toUnmount {
+		if err := mount.UnmountAll(mountPath, unix.MNT_DETACH); err != nil {
+			if i == len(toUnmount)-1 { // last mount
+				return err
+			}
+			// This is some submount, we can ignore this error for now, the final unmount will fail if this is a real problem
+			log.G(ctx).WithError(err).Debugf("failed to unmount submount %s", mountPath)
+		}
+	}
+	return nil
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+//
+// Because of the way `os.Remove` (and by extension `os.RemoveAll`) works, there
+// can be a race between reading directory entries and then actually attempting
+// to remove everything in the directory.
+// These types of errors do not need to be returned since it's ok for the dir to
+// be gone we can just retry the remove operation.
+//
+// This should not return a `os.ErrNotExist` kind of error under any circumstances
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	notExistErr := make(map[string]bool)
+
+	// track retries
+	exitOnErr := make(map[string]int)
+	maxRetry := 50
+
+	// Attempt to unmount anything beneath this dir first.
+	if err := unmountRecursive(ctx, dir); err != nil {
+		log.G(ctx).WithError(err).Debugf("failed to do initial unmount of %s", dir)
+	}
+
+	for {
+		err := os.RemoveAll(dir)
+		if err == nil {
+			return nil
+		}
+
+		pe, ok := err.(*os.PathError)
+		if !ok {
+			return err
+		}
+
+		if os.IsNotExist(err) {
+			if notExistErr[pe.Path] {
+				return err
+			}
+			notExistErr[pe.Path] = true
+
+			// There is a race where some subdir can be removed but after the
+			// parent dir entries have been read.
+			// So the path could be from `os.Remove(subdir)`
+			// If the reported non-existent path is not the passed in `dir` we
+			// should just retry, but otherwise return with no error.
+			if pe.Path == dir {
+				return nil
+			}
+			continue
+		}
+
+		if pe.Err != syscall.EBUSY {
+			return err
+		}
+		if e := mount.Unmount(pe.Path, unix.MNT_DETACH); e != nil {
+			return errors.Wrapf(e, "error while removing %s", dir)
+		}
+
+		if exitOnErr[pe.Path] == maxRetry {
+			return err
+		}
+		exitOnErr[pe.Path]++
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+var vmbasedRuntimes = []string{
+	"io.containerd.kata",
+}
+
+func isVMBasedRuntime(runtimeType string) bool {
+	for _, rt := range vmbasedRuntimes {
+		if strings.Contains(runtimeType, rt) {
+			return true
+		}
+	}
+	return false
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	if !isVMBasedRuntime(runtimeType) {
+		return nil
+	}
+	l, err := getKVMLabel(spec.Process.SelinuxLabel)
+	if err != nil {
+		return errors.Wrap(err, "failed to get selinux kvm label")
+	}
+	spec.Process.SelinuxLabel = l
+	return nil
+}
+
+func getKVMLabel(l string) (string, error) {
+	if !seutil.HasType("container_kvm_t") {
+		return "", nil
+	}
+	return seutil.ChangeToKVM(l)
+}

pkg/cri/server/helpers_linux_test.go

@@ -0,0 +1,106 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+	"golang.org/x/sys/unix"
+)
+
+func TestGetCgroupsPath(t *testing.T) {
+	testID := "test-id"
+	for desc, test := range map[string]struct {
+		cgroupsParent string
+		expected      string
+	}{
+		"should support regular cgroup path": {
+			cgroupsParent: "/a/b",
+			expected:      "/a/b/test-id",
+		},
+		"should support systemd cgroup path": {
+			cgroupsParent: "/a.slice/b.slice",
+			expected:      "b.slice:cri-containerd:test-id",
+		},
+		"should support tailing slash for regular cgroup path": {
+			cgroupsParent: "/a/b/",
+			expected:      "/a/b/test-id",
+		},
+		"should support tailing slash for systemd cgroup path": {
+			cgroupsParent: "/a.slice/b.slice/",
+			expected:      "b.slice:cri-containerd:test-id",
+		},
+		"should treat root cgroup as regular cgroup path": {
+			cgroupsParent: "/",
+			expected:      "/test-id",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got := getCgroupsPath(test.cgroupsParent, testID)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestEnsureRemoveAllWithMount(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("skipping test that requires root")
+	}
+
+	dir1, err := ioutil.TempDir("", "test-ensure-removeall-with-dir1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir2, err := ioutil.TempDir("", "test-ensure-removeall-with-dir2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir2)
+
+	bindDir := filepath.Join(dir1, "bind")
+	if err := os.MkdirAll(bindDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := unix.Mount(dir2, bindDir, "none", unix.MS_BIND, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		err = ensureRemoveAll(context.Background(), dir1)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("timeout waiting for EnsureRemoveAll to finish")
+	}
+
+	if _, err := os.Stat(dir1); !os.IsNotExist(err) {
+		t.Fatalf("expected %q to not exist", dir1)
+	}
+}

pkg/cri/server/helpers_other.go

@@ -0,0 +1,43 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// openLogFile opens/creates a container log file.
+func openLogFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/cri/server/helpers_selinux_test.go

@@ -0,0 +1,159 @@
+// +build selinux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func TestInitSelinuxOpts(t *testing.T) {
+	if !selinux.GetEnabled() {
+		t.Skip("selinux is not enabled")
+	}
+
+	for desc, test := range map[string]struct {
+		selinuxOpt   *runtime.SELinuxOption
+		processLabel string
+		mountLabel   string
+		expectErr    bool
+	}{
+		"Should return empty strings for processLabel and mountLabel when selinuxOpt is nil": {
+			selinuxOpt:   nil,
+			processLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}",
+			mountLabel:   ".*:c[0-9]{1,3},c[0-9]{1,3}",
+		},
+		"Should overlay fields on processLabel when selinuxOpt has been initialized partially": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "",
+				Role:  "user_r",
+				Type:  "",
+				Level: "s0:c1,c2",
+			},
+			processLabel: "system_u:user_r:(container_file_t|svirt_lxc_net_t):s0:c1,c2",
+			mountLabel:   "system_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
+		},
+		"Should be resolved correctly when selinuxOpt has been initialized completely": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "s0:c1,c2",
+			},
+			processLabel: "user_u:user_r:user_t:s0:c1,c2",
+			mountLabel:   "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
+		},
+		"Should be resolved correctly when selinuxOpt has been initialized with level=''": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "",
+			},
+			processLabel: "user_u:user_r:user_t:s0:c[0-9]{1,3},c[0-9]{1,3}",
+			mountLabel:   "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0",
+		},
+		"Should return error when the format of 'level' is not correct": {
+			selinuxOpt: &runtime.SELinuxOption{
+				User:  "user_u",
+				Role:  "user_r",
+				Type:  "user_t",
+				Level: "s0,c1,c2",
+			},
+			expectErr: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			processLabel, mountLabel, err := initLabelsFromOpt(test.selinuxOpt)
+			if test.expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.Regexp(t, test.processLabel, processLabel)
+				assert.Regexp(t, test.mountLabel, mountLabel)
+			}
+		})
+	}
+}
+
+func TestCheckSelinuxLevel(t *testing.T) {
+	for desc, test := range map[string]struct {
+		level         string
+		expectNoMatch bool
+	}{
+		"s0": {
+			level: "s0",
+		},
+		"s0-s0": {
+			level: "s0-s0",
+		},
+		"s0:c0": {
+			level: "s0:c0",
+		},
+		"s0:c0.c3": {
+			level: "s0:c0.c3",
+		},
+		"s0:c0,c3": {
+			level: "s0:c0,c3",
+		},
+		"s0-s0:c0,c3": {
+			level: "s0-s0:c0,c3",
+		},
+		"s0-s0:c0,c3.c6": {
+			level: "s0-s0:c0,c3.c6",
+		},
+		"s0-s0:c0,c3.c6,c8.c10": {
+			level: "s0-s0:c0,c3.c6,c8.c10",
+		},
+		"s0-s0:c0,c3.c6,c8,c10": {
+			level: "s0-s0:c0,c3.c6",
+		},
+		"s0,c0,c3": {
+			level:         "s0,c0,c3",
+			expectNoMatch: true,
+		},
+		"s0:c0.c3.c6": {
+			level:         "s0:c0.c3.c6",
+			expectNoMatch: true,
+		},
+		"s0-s0,c0,c3": {
+			level:         "s0-s0,c0,c3",
+			expectNoMatch: true,
+		},
+		"s0-s0:c0.c3.c6": {
+			level:         "s0-s0:c0.c3.c6",
+			expectNoMatch: true,
+		},
+		"s0-s0:c0,c3.c6.c8": {
+			level:         "s0-s0:c0,c3.c6.c8",
+			expectNoMatch: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			err := checkSelinuxLevel(test.level)
+			if test.expectNoMatch {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

pkg/cri/server/helpers_test.go

@@ -0,0 +1,498 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	imagedigest "github.com/opencontainers/go-digest"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	"github.com/containerd/containerd/pkg/cri/store"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+)
+
+// TestGetUserFromImage tests the logic of getting image uid or user name of image user.
+func TestGetUserFromImage(t *testing.T) {
+	newI64 := func(i int64) *int64 { return &i }
+	for c, test := range map[string]struct {
+		user string
+		uid  *int64
+		name string
+	}{
+		"no gid": {
+			user: "0",
+			uid:  newI64(0),
+		},
+		"uid/gid": {
+			user: "0:1",
+			uid:  newI64(0),
+		},
+		"empty user": {
+			user: "",
+		},
+		"multiple spearators": {
+			user: "1:2:3",
+			uid:  newI64(1),
+		},
+		"root username": {
+			user: "root:root",
+			name: "root",
+		},
+		"username": {
+			user: "test:test",
+			name: "test",
+		},
+	} {
+		t.Logf("TestCase - %q", c)
+		actualUID, actualName := getUserFromImage(test.user)
+		assert.Equal(t, test.uid, actualUID)
+		assert.Equal(t, test.name, actualName)
+	}
+}
+
+func TestGetRepoDigestAndTag(t *testing.T) {
+	digest := imagedigest.Digest("sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582")
+	for desc, test := range map[string]struct {
+		ref                string
+		schema1            bool
+		expectedRepoDigest string
+		expectedRepoTag    string
+	}{
+		"repo tag should be empty if original ref has no tag": {
+			ref:                "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+		},
+		"repo tag should not be empty if original ref has tag": {
+			ref:                "gcr.io/library/busybox:latest",
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should be empty if original ref is schema1 and has no digest": {
+			ref:                "gcr.io/library/busybox:latest",
+			schema1:            true,
+			expectedRepoDigest: "",
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should not be empty if orignal ref is schema1 but has digest": {
+			ref:                "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			schema1:            true,
+			expectedRepoDigest: "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			expectedRepoTag:    "",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		named, err := docker.ParseDockerRef(test.ref)
+		assert.NoError(t, err)
+		repoDigest, repoTag := getRepoDigestAndTag(named, digest, test.schema1)
+		assert.Equal(t, test.expectedRepoDigest, repoDigest)
+		assert.Equal(t, test.expectedRepoTag, repoTag)
+	}
+}
+
+func TestBuildLabels(t *testing.T) {
+	configLabels := map[string]string{
+		"a": "b",
+		"c": "d",
+	}
+	newLabels := buildLabels(configLabels, containerKindSandbox)
+	assert.Len(t, newLabels, 3)
+	assert.Equal(t, "b", newLabels["a"])
+	assert.Equal(t, "d", newLabels["c"])
+	assert.Equal(t, containerKindSandbox, newLabels[containerKindLabel])
+
+	newLabels["a"] = "e"
+	assert.Empty(t, configLabels[containerKindLabel], "should not add new labels into original label")
+	assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
+}
+
+func TestParseImageReferences(t *testing.T) {
+	refs := []string{
+		"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"gcr.io/library/busybox:1.2",
+		"sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"arbitrary-ref",
+	}
+	expectedTags := []string{
+		"gcr.io/library/busybox:1.2",
+	}
+	expectedDigests := []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"}
+	tags, digests := parseImageReferences(refs)
+	assert.Equal(t, expectedTags, tags)
+	assert.Equal(t, expectedDigests, digests)
+}
+
+func TestLocalResolve(t *testing.T) {
+	image := imagestore.Image{
+		ID:      "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		ChainID: "test-chain-id-1",
+		References: []string{
+			"docker.io/library/busybox:latest",
+			"docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+		Size: 10,
+	}
+	c := newTestCRIService()
+	var err error
+	c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{image})
+	assert.NoError(t, err)
+
+	for _, ref := range []string{
+		"sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113799",
+		"busybox",
+		"busybox:latest",
+		"busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"library/busybox",
+		"library/busybox:latest",
+		"library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"docker.io/busybox",
+		"docker.io/busybox:latest",
+		"docker.io/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"docker.io/library/busybox",
+		"docker.io/library/busybox:latest",
+		"docker.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+	} {
+		img, err := c.localResolve(ref)
+		assert.NoError(t, err)
+		assert.Equal(t, image, img)
+	}
+	img, err := c.localResolve("randomid")
+	assert.Equal(t, store.ErrNotExist, err)
+	assert.Equal(t, imagestore.Image{}, img)
+}
+
+func TestGenerateRuntimeOptions(t *testing.T) {
+	nilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+`
+	nonNilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.legacy.options]
+  Runtime = "legacy"
+  RuntimeRoot = "/legacy"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runc.options]
+  BinaryName = "runc"
+  Root = "/runc"
+  NoNewKeyring = true
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+[containerd.runtimes.runcv2.options]
+  BinaryName = "runc"
+  Root = "/runcv2"
+  NoNewKeyring = true
+`
+	var nilOptsConfig, nonNilOptsConfig criconfig.Config
+	_, err := toml.Decode(nilOpts, &nilOptsConfig)
+	require.NoError(t, err)
+	_, err = toml.Decode(nonNilOpts, &nonNilOptsConfig)
+	require.NoError(t, err)
+	require.Len(t, nilOptsConfig.Runtimes, 3)
+	require.Len(t, nonNilOptsConfig.Runtimes, 3)
+
+	for desc, test := range map[string]struct {
+		r               criconfig.Runtime
+		c               criconfig.Config
+		expectedOptions interface{}
+	}{
+		"when options is nil, should return nil option for io.containerd.runc.v1": {
+			r:               nilOptsConfig.Runtimes["runc"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should return nil option for io.containerd.runc.v2": {
+			r:               nilOptsConfig.Runtimes["runcv2"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should use legacy fields for legacy runtime": {
+			r: nilOptsConfig.Runtimes["legacy"],
+			c: nilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				SystemdCgroup: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v1": {
+			r: nonNilOptsConfig.Runtimes["runc"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runc",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v2": {
+			r: nonNilOptsConfig.Runtimes["runcv2"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runcv2",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for legacy runtime": {
+			r: nonNilOptsConfig.Runtimes["legacy"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				Runtime:     "legacy",
+				RuntimeRoot: "/legacy",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			opts, err := generateRuntimeOptions(test.r, test.c)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedOptions, opts)
+		})
+	}
+}
+
+func TestEnvDeduplication(t *testing.T) {
+	for desc, test := range map[string]struct {
+		existing []string
+		kv       [][2]string
+		expected []string
+	}{
+		"single env": {
+			kv: [][2]string{
+				{"a", "b"},
+			},
+			expected: []string{"a=b"},
+		},
+		"multiple envs": {
+			kv: [][2]string{
+				{"a", "b"},
+				{"c", "d"},
+				{"e", "f"},
+			},
+			expected: []string{
+				"a=b",
+				"c=d",
+				"e=f",
+			},
+		},
+		"env override": {
+			kv: [][2]string{
+				{"k1", "v1"},
+				{"k2", "v2"},
+				{"k3", "v3"},
+				{"k3", "v4"},
+				{"k1", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v5",
+				"k2=v2",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+		"existing env": {
+			existing: []string{
+				"k1=v1",
+				"k2=v2",
+				"k3=v3",
+			},
+			kv: [][2]string{
+				{"k3", "v4"},
+				{"k2", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v1",
+				"k2=v5",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		var spec runtimespec.Spec
+		if len(test.existing) > 0 {
+			spec.Process = &runtimespec.Process{
+				Env: test.existing,
+			}
+		}
+		for _, kv := range test.kv {
+			oci.WithEnv([]string{kv[0] + "=" + kv[1]})(context.Background(), nil, nil, &spec)
+		}
+		assert.Equal(t, test.expected, spec.Process.Env)
+	}
+}
+
+func TestPassThroughAnnotationsFilter(t *testing.T) {
+	for desc, test := range map[string]struct {
+		podAnnotations         map[string]string
+		runtimePodAnnotations  []string
+		passthroughAnnotations map[string]string
+	}{
+		"should support direct match": {
+			podAnnotations:         map[string]string{"c": "d", "d": "e"},
+			runtimePodAnnotations:  []string{"c"},
+			passthroughAnnotations: map[string]string{"c": "d"},
+		},
+		"should support wildcard match": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*.f", "z*g", "y.c*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"y.ca": "b",
+			},
+		},
+		"should support wildcard match all": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+		},
+		"should support match including path separator": {
+			podAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchend.com/1end":   "3",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchmid.com/mid1":   "6",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchhead.com/head1": "9",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"1end/matchend":       "14",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"mid1/matchmid":       "17",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"head1/matchhead":     "20",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+				"nomatch/nomatch":     "24",
+			},
+			runtimePodAnnotations: []string{
+				"matchend.com/end*",
+				"matchmid.com/mi*d",
+				"matchhead.com/*head",
+				"matchall.com/*",
+				"end*/matchend",
+				"mi*d/matchmid",
+				"*head/matchhead",
+				"*/matchall",
+				"match*/match*",
+			},
+			passthroughAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			passthroughAnnotations := getPassthroughAnnotations(test.podAnnotations, test.runtimePodAnnotations)
+			assert.Equal(t, test.passthroughAnnotations, passthroughAnnotations)
+		})
+	}
+}
+
+func TestEnsureRemoveAllNotExist(t *testing.T) {
+	// should never return an error for a non-existent path
+	if err := ensureRemoveAll(context.Background(), "/non/existent/path"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithDir(t *testing.T) {
+	dir, err := ioutil.TempDir("", "test-ensure-removeall-with-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ensureRemoveAll(context.Background(), dir); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithFile(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "test-ensure-removeall-with-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmp.Close()
+	if err := ensureRemoveAll(context.Background(), tmp.Name()); err != nil {
+		t.Fatal(err)
+	}
+}

pkg/cri/server/helpers_windows.go

@@ -0,0 +1,170 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// openLogFile opens/creates a container log file.
+// It specifies `FILE_SHARE_DELETE` option to make sure
+// log files can be rotated by kubelet.
+// TODO(windows): Use golang support after 1.14. (https://github.com/golang/go/issues/32088)
+func openLogFile(path string) (*os.File, error) {
+	path = fixLongPath(path)
+	if len(path) == 0 {
+		return nil, syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return nil, err
+	}
+	createmode := uint32(syscall.OPEN_ALWAYS)
+	access := uint32(syscall.FILE_APPEND_DATA)
+	sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
+	h, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+	if err != nil {
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}
+
+// Copyright (c) 2009 The Go Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+//
+// This is copied from https://golang.org/src/path/filepath/path_windows.go.
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less then 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !filepath.IsAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}
+
+// ensureRemoveAll is a wrapper for os.RemoveAll on Windows.
+func ensureRemoveAll(_ context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/cri/server/image_list.go

@@ -0,0 +1,38 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ListImages lists existing images.
+// TODO(random-liu): Add image list filters after CRI defines this more clear, and kubelet
+// actually needs it.
+func (c *criService) ListImages(ctx context.Context, r *runtime.ListImagesRequest) (*runtime.ListImagesResponse, error) {
+	imagesInStore := c.imageStore.List()
+
+	var images []*runtime.Image
+	for _, image := range imagesInStore {
+		// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
+		// doesn't exist?
+		images = append(images, toCRIImage(image))
+	}
+
+	return &runtime.ListImagesResponse{Images: images}, nil
+}

pkg/cri/server/image_list_test.go

@@ -0,0 +1,113 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+)
+
+func TestListImages(t *testing.T) {
+	c := newTestCRIService()
+	imagesInStore := []imagestore.Image{
+		{
+			ID:      "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-1",
+			References: []string{
+				"gcr.io/library/busybox:latest",
+				"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 1000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "root",
+				},
+			},
+		},
+		{
+			ID:      "sha256:2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-2",
+			References: []string{
+				"gcr.io/library/alpine:latest",
+				"gcr.io/library/alpine@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 2000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "1234:1234",
+				},
+			},
+		},
+		{
+			ID:      "sha256:3123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			ChainID: "test-chainid-3",
+			References: []string{
+				"gcr.io/library/ubuntu:latest",
+				"gcr.io/library/ubuntu@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+			},
+			Size: 3000,
+			ImageSpec: imagespec.Image{
+				Config: imagespec.ImageConfig{
+					User: "nobody",
+				},
+			},
+		},
+	}
+	expect := []*runtime.Image{
+		{
+			Id:          "sha256:1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/busybox:latest"},
+			RepoDigests: []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(1000),
+			Username:    "root",
+		},
+		{
+			Id:          "sha256:2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/alpine:latest"},
+			RepoDigests: []string{"gcr.io/library/alpine@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(2000),
+			Uid:         &runtime.Int64Value{Value: 1234},
+		},
+		{
+			Id:          "sha256:3123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+			RepoTags:    []string{"gcr.io/library/ubuntu:latest"},
+			RepoDigests: []string{"gcr.io/library/ubuntu@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+			Size_:       uint64(3000),
+			Username:    "nobody",
+		},
+	}
+
+	var err error
+	c.imageStore, err = imagestore.NewFakeStore(imagesInStore)
+	assert.NoError(t, err)
+
+	resp, err := c.ListImages(context.Background(), &runtime.ListImagesRequest{})
+	assert.NoError(t, err)
+	require.NotNil(t, resp)
+	images := resp.GetImages()
+	assert.Len(t, images, len(expect))
+	for _, i := range expect {
+		assert.Contains(t, images, i)
+	}
+}

pkg/cri/server/image_pull.go

@@ -0,0 +1,519 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/log"
+	distribution "github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/imgcrypt"
+	"github.com/containerd/imgcrypt/images/encryption"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+)
+
+// For image management:
+// 1) We have an in-memory metadata index to:
+//   a. Maintain ImageID -> RepoTags, ImageID -> RepoDigset relationships; ImageID
+//   is the digest of image config, which conforms to oci image spec.
+//   b. Cache constant and useful information such as image chainID, config etc.
+//   c. An image will be added into the in-memory metadata only when it's successfully
+//   pulled and unpacked.
+//
+// 2) We use containerd image metadata store and content store:
+//   a. To resolve image reference (digest/tag) locally. During pulling image, we
+//   normalize the image reference provided by user, and put it into image metadata
+//   store with resolved descriptor. For the other operations, if image id is provided,
+//   we'll access the in-memory metadata index directly; if image reference is
+//   provided, we'll normalize it, resolve it in containerd image metadata store
+//   to get the image id.
+//   b. As the backup of in-memory metadata in 1). During startup, the in-memory
+//   metadata could be re-constructed from image metadata store + content store.
+//
+// Several problems with current approach:
+// 1) An entry in containerd image metadata store doesn't mean a "READY" (successfully
+// pulled and unpacked) image. E.g. during pulling, the client gets killed. In that case,
+// if we saw an image without snapshots or with in-complete contents during startup,
+// should we re-pull the image? Or should we remove the entry?
+//
+// yanxuean: We can't delete image directly, because we don't know if the image
+// is pulled by us. There are resource leakage.
+//
+// 2) Containerd suggests user to add entry before pulling the image. However if
+// an error occurs during the pulling, should we remove the entry from metadata
+// store? Or should we leave it there until next startup (resource leakage)?
+//
+// 3) The cri plugin only exposes "READY" (successfully pulled and unpacked) images
+// to the user, which are maintained in the in-memory metadata index. However, it's
+// still possible that someone else removes the content or snapshot by-pass the cri plugin,
+// how do we detect that and update the in-memory metadata correspondingly? Always
+// check whether corresponding snapshot is ready when reporting image status?
+//
+// 4) Is the content important if we cached necessary information in-memory
+// after we pull the image? How to manage the disk usage of contents? If some
+// contents are missing but snapshots are ready, is the image still "READY"?
+
+// PullImage pulls an image with authentication config.
+func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
+	imageRef := r.GetImage().GetImage()
+	namedRef, err := distribution.ParseDockerRef(imageRef)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse image reference %q", imageRef)
+	}
+	ref := namedRef.String()
+	if ref != imageRef {
+		log.G(ctx).Debugf("PullImage using normalized image ref: %q", ref)
+	}
+	var (
+		resolver = docker.NewResolver(docker.ResolverOptions{
+			Headers: c.config.Registry.Headers,
+			Hosts:   c.registryHosts(r.GetAuth()),
+		})
+		isSchema1    bool
+		imageHandler containerdimages.HandlerFunc = func(_ context.Context,
+			desc imagespec.Descriptor) ([]imagespec.Descriptor, error) {
+			if desc.MediaType == containerdimages.MediaTypeDockerSchema1Manifest {
+				isSchema1 = true
+			}
+			return nil, nil
+		}
+	)
+
+	pullOpts := []containerd.RemoteOpt{
+		containerd.WithSchema1Conversion,
+		containerd.WithResolver(resolver),
+		containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter),
+		containerd.WithPullUnpack,
+		containerd.WithPullLabel(imageLabelKey, imageLabelValue),
+		containerd.WithMaxConcurrentDownloads(c.config.MaxConcurrentDownloads),
+		containerd.WithImageHandler(imageHandler),
+	}
+
+	pullOpts = append(pullOpts, c.encryptedImagesPullOpts()...)
+	if !c.config.ContainerdConfig.DisableSnapshotAnnotations {
+		pullOpts = append(pullOpts,
+			containerd.WithImageHandlerWrapper(appendInfoHandlerWrapper(ref)))
+	}
+
+	if c.config.ContainerdConfig.DiscardUnpackedLayers {
+		// Allows GC to clean layers up from the content store after unpacking
+		pullOpts = append(pullOpts,
+			containerd.WithChildLabelMap(containerdimages.ChildGCLabelsFilterLayers))
+	}
+
+	image, err := c.client.Pull(ctx, ref, pullOpts...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to pull and unpack image %q", ref)
+	}
+
+	configDesc, err := image.Config(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "get image config descriptor")
+	}
+	imageID := configDesc.Digest.String()
+
+	repoDigest, repoTag := getRepoDigestAndTag(namedRef, image.Target().Digest, isSchema1)
+	for _, r := range []string{imageID, repoTag, repoDigest} {
+		if r == "" {
+			continue
+		}
+		if err := c.createImageReference(ctx, r, image.Target()); err != nil {
+			return nil, errors.Wrapf(err, "failed to create image reference %q", r)
+		}
+		// Update image store to reflect the newest state in containerd.
+		// No need to use `updateImage`, because the image reference must
+		// have been managed by the cri plugin.
+		if err := c.imageStore.Update(ctx, r); err != nil {
+			return nil, errors.Wrapf(err, "failed to update image store %q", r)
+		}
+	}
+
+	log.G(ctx).Debugf("Pulled image %q with image id %q, repo tag %q, repo digest %q", imageRef, imageID,
+		repoTag, repoDigest)
+	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
+	// in-memory image store, it's only for in-memory indexing. The image could be removed
+	// by someone else anytime, before/during/after we create the metadata. We should always
+	// check the actual state in containerd before using the image or returning status of the
+	// image.
+	return &runtime.PullImageResponse{ImageRef: imageID}, nil
+}
+
+// ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
+func ParseAuth(auth *runtime.AuthConfig, host string) (string, string, error) {
+	if auth == nil {
+		return "", "", nil
+	}
+	if auth.ServerAddress != "" {
+		// Do not return the auth info when server address doesn't match.
+		u, err := url.Parse(auth.ServerAddress)
+		if err != nil {
+			return "", "", errors.Wrap(err, "parse server address")
+		}
+		if host != u.Host {
+			return "", "", nil
+		}
+	}
+	if auth.Username != "" {
+		return auth.Username, auth.Password, nil
+	}
+	if auth.IdentityToken != "" {
+		return "", auth.IdentityToken, nil
+	}
+	if auth.Auth != "" {
+		decLen := base64.StdEncoding.DecodedLen(len(auth.Auth))
+		decoded := make([]byte, decLen)
+		_, err := base64.StdEncoding.Decode(decoded, []byte(auth.Auth))
+		if err != nil {
+			return "", "", err
+		}
+		fields := strings.SplitN(string(decoded), ":", 2)
+		if len(fields) != 2 {
+			return "", "", errors.Errorf("invalid decoded auth: %q", decoded)
+		}
+		user, passwd := fields[0], fields[1]
+		return user, strings.Trim(passwd, "\x00"), nil
+	}
+	// TODO(random-liu): Support RegistryToken.
+	// An empty auth config is valid for anonymous registry
+	return "", "", nil
+}
+
+// createImageReference creates image reference inside containerd image store.
+// Note that because create and update are not finished in one transaction, there could be race. E.g.
+// the image reference is deleted by someone else after create returns already exists, but before update
+// happens.
+func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
+	img := containerdimages.Image{
+		Name:   name,
+		Target: desc,
+		// Add a label to indicate that the image is managed by the cri plugin.
+		Labels: map[string]string{imageLabelKey: imageLabelValue},
+	}
+	// TODO(random-liu): Figure out which is the more performant sequence create then update or
+	// update then create.
+	oldImg, err := c.client.ImageService().Create(ctx, img)
+	if err == nil || !errdefs.IsAlreadyExists(err) {
+		return err
+	}
+	if oldImg.Target.Digest == img.Target.Digest && oldImg.Labels[imageLabelKey] == imageLabelValue {
+		return nil
+	}
+	_, err = c.client.ImageService().Update(ctx, img, "target", "labels")
+	return err
+}
+
+// updateImage updates image store to reflect the newest state of an image reference
+// in containerd. If the reference is not managed by the cri plugin, the function also
+// generates necessary metadata for the image and make it managed.
+func (c *criService) updateImage(ctx context.Context, r string) error {
+	img, err := c.client.GetImage(ctx, r)
+	if err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrap(err, "get image by reference")
+	}
+	if err == nil && img.Labels()[imageLabelKey] != imageLabelValue {
+		// Make sure the image has the image id as its unique
+		// identifier that references the image in its lifetime.
+		configDesc, err := img.Config(ctx)
+		if err != nil {
+			return errors.Wrap(err, "get image id")
+		}
+		id := configDesc.Digest.String()
+		if err := c.createImageReference(ctx, id, img.Target()); err != nil {
+			return errors.Wrapf(err, "create image id reference %q", id)
+		}
+		if err := c.imageStore.Update(ctx, id); err != nil {
+			return errors.Wrapf(err, "update image store for %q", id)
+		}
+		// The image id is ready, add the label to mark the image as managed.
+		if err := c.createImageReference(ctx, r, img.Target()); err != nil {
+			return errors.Wrap(err, "create managed label")
+		}
+	}
+	// If the image is not found, we should continue updating the cache,
+	// so that the image can be removed from the cache.
+	if err := c.imageStore.Update(ctx, r); err != nil {
+		return errors.Wrapf(err, "update image store for %q", r)
+	}
+	return nil
+}
+
+// getTLSConfig returns a TLSConfig configured with a CA/Cert/Key specified by registryTLSConfig
+func (c *criService) getTLSConfig(registryTLSConfig criconfig.TLSConfig) (*tls.Config, error) {
+	var (
+		tlsConfig = &tls.Config{}
+		cert      tls.Certificate
+		err       error
+	)
+	if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile == "" {
+		return nil, errors.Errorf("cert file %q was specified, but no corresponding key file was specified", registryTLSConfig.CertFile)
+	}
+	if registryTLSConfig.CertFile == "" && registryTLSConfig.KeyFile != "" {
+		return nil, errors.Errorf("key file %q was specified, but no corresponding cert file was specified", registryTLSConfig.KeyFile)
+	}
+	if registryTLSConfig.CertFile != "" && registryTLSConfig.KeyFile != "" {
+		cert, err = tls.LoadX509KeyPair(registryTLSConfig.CertFile, registryTLSConfig.KeyFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load cert file")
+		}
+		if len(cert.Certificate) != 0 {
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+		tlsConfig.BuildNameToCertificate() // nolint:staticcheck
+	}
+
+	if registryTLSConfig.CAFile != "" {
+		caCertPool, err := x509.SystemCertPool()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get system cert pool")
+		}
+		caCert, err := ioutil.ReadFile(registryTLSConfig.CAFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load CA file")
+		}
+		caCertPool.AppendCertsFromPEM(caCert)
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	tlsConfig.InsecureSkipVerify = registryTLSConfig.InsecureSkipVerify
+	return tlsConfig, nil
+}
+
+// registryHosts is the registry hosts to be used by the resolver.
+func (c *criService) registryHosts(auth *runtime.AuthConfig) docker.RegistryHosts {
+	return func(host string) ([]docker.RegistryHost, error) {
+		var registries []docker.RegistryHost
+
+		endpoints, err := c.registryEndpoints(host)
+		if err != nil {
+			return nil, errors.Wrap(err, "get registry endpoints")
+		}
+		for _, e := range endpoints {
+			u, err := url.Parse(e)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parse registry endpoint %q from mirrors", e)
+			}
+
+			var (
+				transport = newTransport()
+				client    = &http.Client{Transport: transport}
+				config    = c.config.Registry.Configs[u.Host]
+			)
+
+			if config.TLS != nil {
+				transport.TLSClientConfig, err = c.getTLSConfig(*config.TLS)
+				if err != nil {
+					return nil, errors.Wrapf(err, "get TLSConfig for registry %q", e)
+				}
+			}
+
+			if auth == nil && config.Auth != nil {
+				auth = toRuntimeAuthConfig(*config.Auth)
+			}
+
+			if u.Path == "" {
+				u.Path = "/v2"
+			}
+
+			registries = append(registries, docker.RegistryHost{
+				Client: client,
+				Authorizer: docker.NewDockerAuthorizer(
+					docker.WithAuthClient(client),
+					docker.WithAuthCreds(func(host string) (string, string, error) {
+						return ParseAuth(auth, host)
+					})),
+				Host:         u.Host,
+				Scheme:       u.Scheme,
+				Path:         u.Path,
+				Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull,
+			})
+		}
+		return registries, nil
+	}
+}
+
+// defaultScheme returns the default scheme for a registry host.
+func defaultScheme(host string) string {
+	if h, _, err := net.SplitHostPort(host); err == nil {
+		host = h
+	}
+	if host == "localhost" || host == "127.0.0.1" || host == "::1" {
+		return "http"
+	}
+	return "https"
+}
+
+// addDefaultScheme returns the endpoint with default scheme
+func addDefaultScheme(endpoint string) (string, error) {
+	if strings.Contains(endpoint, "://") {
+		return endpoint, nil
+	}
+	ue := "dummy://" + endpoint
+	u, err := url.Parse(ue)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%s://%s", defaultScheme(u.Host), endpoint), nil
+}
+
+// registryEndpoints returns endpoints for a given host.
+// It adds default registry endpoint if it does not exist in the passed-in endpoint list.
+// It also supports wildcard host matching with `*`.
+func (c *criService) registryEndpoints(host string) ([]string, error) {
+	var endpoints []string
+	_, ok := c.config.Registry.Mirrors[host]
+	if ok {
+		endpoints = c.config.Registry.Mirrors[host].Endpoints
+	} else {
+		endpoints = c.config.Registry.Mirrors["*"].Endpoints
+	}
+	defaultHost, err := docker.DefaultHost(host)
+	if err != nil {
+		return nil, errors.Wrap(err, "get default host")
+	}
+	for i := range endpoints {
+		en, err := addDefaultScheme(endpoints[i])
+		if err != nil {
+			return nil, errors.Wrap(err, "parse endpoint url")
+		}
+		endpoints[i] = en
+	}
+	for _, e := range endpoints {
+		u, err := url.Parse(e)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse endpoint url")
+		}
+		if u.Host == host {
+			// Do not add default if the endpoint already exists.
+			return endpoints, nil
+		}
+	}
+	return append(endpoints, defaultScheme(defaultHost)+"://"+defaultHost), nil
+}
+
+// newTransport returns a new HTTP transport used to pull image.
+// TODO(random-liu): Create a library and share this code with `ctr`.
+func newTransport() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:       30 * time.Second,
+			KeepAlive:     30 * time.Second,
+			FallbackDelay: 300 * time.Millisecond,
+		}).DialContext,
+		MaxIdleConns:          10,
+		IdleConnTimeout:       30 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+	}
+}
+
+// encryptedImagesPullOpts returns the necessary list of pull options required
+// for decryption of encrypted images based on the cri decryption configuration.
+func (c *criService) encryptedImagesPullOpts() []containerd.RemoteOpt {
+	if c.config.ImageDecryption.KeyModel == criconfig.KeyModelNode {
+		ltdd := imgcrypt.Payload{}
+		decUnpackOpt := encryption.WithUnpackConfigApplyOpts(encryption.WithDecryptedUnpack(&ltdd))
+		opt := containerd.WithUnpackOpts([]containerd.UnpackOpt{decUnpackOpt})
+		return []containerd.RemoteOpt{opt}
+	}
+	return nil
+}
+
+const (
+	// targetRefLabel is a label which contains image reference and will be passed
+	// to snapshotters.
+	targetRefLabel = "containerd.io/snapshot/cri.image-ref"
+	// targetDigestLabel is a label which contains layer digest and will be passed
+	// to snapshotters.
+	targetDigestLabel = "containerd.io/snapshot/cri.layer-digest"
+	// targetImageLayersLabel is a label which contains layer digests contained in
+	// the target image and will be passed to snapshotters for preparing layers in
+	// parallel. Skipping some layers is allowed and only affects performance.
+	targetImageLayersLabel = "containerd.io/snapshot/cri.image-layers"
+)
+
+// appendInfoHandlerWrapper makes a handler which appends some basic information
+// of images to each layer descriptor as annotations during unpack. These
+// annotations will be passed to snapshotters as labels. These labels will be
+// used mainly by stargz-based snapshotters for querying image contents from the
+// registry.
+func appendInfoHandlerWrapper(ref string) func(f containerdimages.Handler) containerdimages.Handler {
+	return func(f containerdimages.Handler) containerdimages.Handler {
+		return containerdimages.HandlerFunc(func(ctx context.Context, desc imagespec.Descriptor) ([]imagespec.Descriptor, error) {
+			children, err := f.Handle(ctx, desc)
+			if err != nil {
+				return nil, err
+			}
+			switch desc.MediaType {
+			case imagespec.MediaTypeImageManifest, containerdimages.MediaTypeDockerSchema2Manifest:
+				for i := range children {
+					c := &children[i]
+					if containerdimages.IsLayerType(c.MediaType) {
+						if c.Annotations == nil {
+							c.Annotations = make(map[string]string)
+						}
+						c.Annotations[targetRefLabel] = ref
+						c.Annotations[targetDigestLabel] = c.Digest.String()
+						c.Annotations[targetImageLayersLabel] = getLayers(ctx, targetImageLayersLabel, children[i:], labels.Validate)
+					}
+				}
+			}
+			return children, nil
+		})
+	}
+}
+
+// getLayers returns comma-separated digests based on the passed list of
+// descriptors. The returned list contains as many digests as possible as well
+// as meets the label validation.
+func getLayers(ctx context.Context, key string, descs []imagespec.Descriptor, validate func(k, v string) error) (layers string) {
+	var item string
+	for _, l := range descs {
+		if containerdimages.IsLayerType(l.MediaType) {
+			item = l.Digest.String()
+			if layers != "" {
+				item = "," + item
+			}
+			// This avoids the label hits the size limitation.
+			if err := validate(key, layers+item); err != nil {
+				log.G(ctx).WithError(err).WithField("label", key).Debugf("%q is omitted in the layers list", l.Digest.String())
+				break
+			}
+			layers += item
+		}
+	}
+	return
+}

pkg/cri/server/image_pull_test.go

@@ -0,0 +1,379 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"testing"
+
+	digest "github.com/opencontainers/go-digest"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+)
+
+func TestParseAuth(t *testing.T) {
+	testUser := "username"
+	testPasswd := "password"
+	testAuthLen := base64.StdEncoding.EncodedLen(len(testUser + ":" + testPasswd))
+	testAuth := make([]byte, testAuthLen)
+	base64.StdEncoding.Encode(testAuth, []byte(testUser+":"+testPasswd))
+	invalidAuth := make([]byte, testAuthLen)
+	base64.StdEncoding.Encode(invalidAuth, []byte(testUser+"@"+testPasswd))
+	for desc, test := range map[string]struct {
+		auth           *runtime.AuthConfig
+		host           string
+		expectedUser   string
+		expectedSecret string
+		expectErr      bool
+	}{
+		"should not return error if auth config is nil": {},
+		"should not return error if empty auth is provided for access to anonymous registry": {
+			auth:      &runtime.AuthConfig{},
+			expectErr: false,
+		},
+		"should support identity token": {
+			auth:           &runtime.AuthConfig{IdentityToken: "abcd"},
+			expectedSecret: "abcd",
+		},
+		"should support username and password": {
+			auth: &runtime.AuthConfig{
+				Username: testUser,
+				Password: testPasswd,
+			},
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should support auth": {
+			auth:           &runtime.AuthConfig{Auth: string(testAuth)},
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should return error for invalid auth": {
+			auth:      &runtime.AuthConfig{Auth: string(invalidAuth)},
+			expectErr: true,
+		},
+		"should return empty auth if server address doesn't match": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-2.io",
+			expectedUser:   "",
+			expectedSecret: "",
+		},
+		"should return auth if server address matches": {
+			auth: &runtime.AuthConfig{
+				Username:      testUser,
+				Password:      testPasswd,
+				ServerAddress: "https://registry-1.io",
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+		"should return auth if server address is not specified": {
+			auth: &runtime.AuthConfig{
+				Username: testUser,
+				Password: testPasswd,
+			},
+			host:           "registry-1.io",
+			expectedUser:   testUser,
+			expectedSecret: testPasswd,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		u, s, err := ParseAuth(test.auth, test.host)
+		assert.Equal(t, test.expectErr, err != nil)
+		assert.Equal(t, test.expectedUser, u)
+		assert.Equal(t, test.expectedSecret, s)
+	}
+}
+
+func TestRegistryEndpoints(t *testing.T) {
+	for desc, test := range map[string]struct {
+		mirrors  map[string]criconfig.Mirror
+		host     string
+		expected []string
+	}{
+		"no mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-1.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-3.io",
+			},
+		},
+		"mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"wildcard mirror configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"*": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"host should take precedence if both host and wildcard mirrors are configured": {
+			mirrors: map[string]criconfig.Mirror{
+				"*": {
+					Endpoints: []string{
+						"https://registry-1.io",
+					},
+				},
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-2.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"default endpoint in list with http": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"http://registry-3.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"http://registry-3.io",
+			},
+		},
+		"default endpoint in list with https": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"https://registry-3.io",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io",
+			},
+		},
+		"default endpoint in list with path": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-1.io",
+						"https://registry-2.io",
+						"https://registry-3.io/path",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-1.io",
+				"https://registry-2.io",
+				"https://registry-3.io/path",
+			},
+		},
+		"miss scheme endpoint in list with path": {
+			mirrors: map[string]criconfig.Mirror{
+				"registry-3.io": {
+					Endpoints: []string{
+						"https://registry-3.io",
+						"registry-1.io",
+						"127.0.0.1:1234",
+					},
+				},
+			},
+			host: "registry-3.io",
+			expected: []string{
+				"https://registry-3.io",
+				"https://registry-1.io",
+				"http://127.0.0.1:1234",
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		c.config.Registry.Mirrors = test.mirrors
+		got, err := c.registryEndpoints(test.host)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestDefaultScheme(t *testing.T) {
+	for desc, test := range map[string]struct {
+		host     string
+		expected string
+	}{
+		"should use http by default for localhost": {
+			host:     "localhost",
+			expected: "http",
+		},
+		"should use http by default for localhost with port": {
+			host:     "localhost:8080",
+			expected: "http",
+		},
+		"should use http by default for 127.0.0.1": {
+			host:     "127.0.0.1",
+			expected: "http",
+		},
+		"should use http by default for 127.0.0.1 with port": {
+			host:     "127.0.0.1:8080",
+			expected: "http",
+		},
+		"should use http by default for ::1": {
+			host:     "::1",
+			expected: "http",
+		},
+		"should use http by default for ::1 with port": {
+			host:     "[::1]:8080",
+			expected: "http",
+		},
+		"should use https by default for remote host": {
+			host:     "remote",
+			expected: "https",
+		},
+		"should use https by default for remote host with port": {
+			host:     "remote:8080",
+			expected: "https",
+		},
+		"should use https by default for remote ip": {
+			host:     "8.8.8.8",
+			expected: "https",
+		},
+		"should use https by default for remote ip with port": {
+			host:     "8.8.8.8:8080",
+			expected: "https",
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		got := defaultScheme(test.host)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestEncryptedImagePullOpts(t *testing.T) {
+	for desc, test := range map[string]struct {
+		keyModel     string
+		expectedOpts int
+	}{
+		"node key model should return one unpack opt": {
+			keyModel:     criconfig.KeyModelNode,
+			expectedOpts: 1,
+		},
+		"no key model selected should default to node key model": {
+			keyModel:     "",
+			expectedOpts: 0,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		c.config.ImageDecryption.KeyModel = test.keyModel
+		got := len(c.encryptedImagesPullOpts())
+		assert.Equal(t, test.expectedOpts, got)
+	}
+}
+
+func TestImageLayersLabel(t *testing.T) {
+	sampleKey := "sampleKey"
+	sampleDigest, err := digest.Parse("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	assert.NoError(t, err)
+	sampleMaxSize := 300
+	sampleValidate := func(k, v string) error {
+		if (len(k) + len(v)) > sampleMaxSize {
+			return fmt.Errorf("invalid: %q: %q", k, v)
+		}
+		return nil
+	}
+
+	tests := []struct {
+		name      string
+		layersNum int
+		wantNum   int
+	}{
+		{
+			name:      "valid number of layers",
+			layersNum: 2,
+			wantNum:   2,
+		},
+		{
+			name:      "many layers",
+			layersNum: 5, // hits sampleMaxSize (300 chars).
+			wantNum:   4, // layers should be omitted for avoiding invalid label.
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var sampleLayers []imagespec.Descriptor
+			for i := 0; i < tt.layersNum; i++ {
+				sampleLayers = append(sampleLayers, imagespec.Descriptor{
+					MediaType: imagespec.MediaTypeImageLayerGzip,
+					Digest:    sampleDigest,
+				})
+			}
+			gotS := getLayers(context.Background(), sampleKey, sampleLayers, sampleValidate)
+			got := len(strings.Split(gotS, ","))
+			assert.Equal(t, tt.wantNum, got)
+		})
+	}
+}

pkg/cri/server/image_remove.go

@@ -0,0 +1,65 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+)
+
+// RemoveImage removes the image.
+// TODO(random-liu): Update CRI to pass image reference instead of ImageSpec. (See
+// kubernetes/kubernetes#46255)
+// TODO(random-liu): We should change CRI to distinguish image id and image spec.
+// Remove the whole image no matter the it's image id or reference. This is the
+// semantic defined in CRI now.
+func (c *criService) RemoveImage(ctx context.Context, r *runtime.RemoveImageRequest) (*runtime.RemoveImageResponse, error) {
+	image, err := c.localResolve(r.GetImage().GetImage())
+	if err != nil {
+		if err == store.ErrNotExist {
+			// return empty without error when image not found.
+			return &runtime.RemoveImageResponse{}, nil
+		}
+		return nil, errors.Wrapf(err, "can not resolve %q locally", r.GetImage().GetImage())
+	}
+
+	// Remove all image references.
+	for i, ref := range image.References {
+		var opts []images.DeleteOpt
+		if i == len(image.References)-1 {
+			// Delete the last image reference synchronously to trigger garbage collection.
+			// This is best effort. It is possible that the image reference is deleted by
+			// someone else before this point.
+			opts = []images.DeleteOpt{images.SynchronousDelete()}
+		}
+		err = c.client.ImageService().Delete(ctx, ref, opts...)
+		if err == nil || errdefs.IsNotFound(err) {
+			// Update image store to reflect the newest state in containerd.
+			if err := c.imageStore.Update(ctx, ref); err != nil {
+				return nil, errors.Wrapf(err, "failed to update image reference %q for %q", ref, image.ID)
+			}
+			continue
+		}
+		return nil, errors.Wrapf(err, "failed to delete image reference %q for %q", ref, image.ID)
+	}
+	return &runtime.RemoveImageResponse{}, nil
+}

pkg/cri/server/image_status.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+
+	"github.com/containerd/containerd/log"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+)
+
+// ImageStatus returns the status of the image, returns nil if the image isn't present.
+// TODO(random-liu): We should change CRI to distinguish image id and image spec. (See
+// kubernetes/kubernetes#46255)
+func (c *criService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (*runtime.ImageStatusResponse, error) {
+	image, err := c.localResolve(r.GetImage().GetImage())
+	if err != nil {
+		if err == store.ErrNotExist {
+			// return empty without error when image not found.
+			return &runtime.ImageStatusResponse{}, nil
+		}
+		return nil, errors.Wrapf(err, "can not resolve %q locally", r.GetImage().GetImage())
+	}
+	// TODO(random-liu): [P0] Make sure corresponding snapshot exists. What if snapshot
+	// doesn't exist?
+
+	runtimeImage := toCRIImage(image)
+	info, err := c.toCRIImageInfo(ctx, &image, r.GetVerbose())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate image info")
+	}
+
+	return &runtime.ImageStatusResponse{
+		Image: runtimeImage,
+		Info:  info,
+	}, nil
+}
+
+// toCRIImage converts internal image object to CRI runtime.Image.
+func toCRIImage(image imagestore.Image) *runtime.Image {
+	repoTags, repoDigests := parseImageReferences(image.References)
+	runtimeImage := &runtime.Image{
+		Id:          image.ID,
+		RepoTags:    repoTags,
+		RepoDigests: repoDigests,
+		Size_:       uint64(image.Size),
+	}
+	uid, username := getUserFromImage(image.ImageSpec.Config.User)
+	if uid != nil {
+		runtimeImage.Uid = &runtime.Int64Value{Value: *uid}
+	}
+	runtimeImage.Username = username
+
+	return runtimeImage
+}
+
+// TODO (mikebrow): discuss moving this struct and / or constants for info map for some or all of these fields to CRI
+type verboseImageInfo struct {
+	ChainID   string          `json:"chainID"`
+	ImageSpec imagespec.Image `json:"imageSpec"`
+}
+
+// toCRIImageInfo converts internal image object information to CRI image status response info map.
+func (c *criService) toCRIImageInfo(ctx context.Context, image *imagestore.Image, verbose bool) (map[string]string, error) {
+	if !verbose {
+		return nil, nil
+	}
+
+	info := make(map[string]string)
+
+	imi := &verboseImageInfo{
+		ChainID:   image.ChainID,
+		ImageSpec: image.ImageSpec,
+	}
+
+	m, err := json.Marshal(imi)
+	if err == nil {
+		info["info"] = string(m)
+	} else {
+		log.G(ctx).WithError(err).Errorf("failed to marshal info %v", imi)
+		info["info"] = err.Error()
+	}
+
+	return info, nil
+}

pkg/cri/server/image_status_test.go

@@ -0,0 +1,74 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+)
+
+func TestImageStatus(t *testing.T) {
+	testID := "sha256:d848ce12891bf78792cda4a23c58984033b0c397a55e93a1556202222ecc5ed4"
+	image := imagestore.Image{
+		ID:      testID,
+		ChainID: "test-chain-id",
+		References: []string{
+			"gcr.io/library/busybox:latest",
+			"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		},
+		Size: 1234,
+		ImageSpec: imagespec.Image{
+			Config: imagespec.ImageConfig{
+				User: "user:group",
+			},
+		},
+	}
+	expected := &runtime.Image{
+		Id:          testID,
+		RepoTags:    []string{"gcr.io/library/busybox:latest"},
+		RepoDigests: []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"},
+		Size_:       uint64(1234),
+		Username:    "user",
+	}
+
+	c := newTestCRIService()
+	t.Logf("should return nil image spec without error for non-exist image")
+	resp, err := c.ImageStatus(context.Background(), &runtime.ImageStatusRequest{
+		Image: &runtime.ImageSpec{Image: testID},
+	})
+	assert.NoError(t, err)
+	require.NotNil(t, resp)
+	assert.Nil(t, resp.GetImage())
+
+	c.imageStore, err = imagestore.NewFakeStore([]imagestore.Image{image})
+	assert.NoError(t, err)
+
+	t.Logf("should return correct image status for exist image")
+	resp, err = c.ImageStatus(context.Background(), &runtime.ImageStatusRequest{
+		Image: &runtime.ImageSpec{Image: testID},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, resp)
+	assert.Equal(t, expected, resp.GetImage())
+}

pkg/cri/server/imagefs_info.go

@@ -0,0 +1,52 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"time"
+
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// ImageFsInfo returns information of the filesystem that is used to store images.
+// TODO(windows): Usage for windows is always 0 right now. Support this for windows.
+func (c *criService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (*runtime.ImageFsInfoResponse, error) {
+	snapshots := c.snapshotStore.List()
+	timestamp := time.Now().UnixNano()
+	var usedBytes, inodesUsed uint64
+	for _, sn := range snapshots {
+		// Use the oldest timestamp as the timestamp of imagefs info.
+		if sn.Timestamp < timestamp {
+			timestamp = sn.Timestamp
+		}
+		usedBytes += sn.Size
+		inodesUsed += sn.Inodes
+	}
+	// TODO(random-liu): Handle content store
+	return &runtime.ImageFsInfoResponse{
+		ImageFilesystems: []*runtime.FilesystemUsage{
+			{
+				Timestamp:  timestamp,
+				FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPath},
+				UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+				InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+			},
+		},
+	}, nil
+}

pkg/cri/server/imagefs_info_test.go

@@ -0,0 +1,70 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	snapshot "github.com/containerd/containerd/snapshots"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	snapshotstore "github.com/containerd/containerd/pkg/cri/store/snapshot"
+)
+
+func TestImageFsInfo(t *testing.T) {
+	c := newTestCRIService()
+	snapshots := []snapshotstore.Snapshot{
+		{
+			Key:       "key1",
+			Kind:      snapshot.KindActive,
+			Size:      10,
+			Inodes:    100,
+			Timestamp: 234567,
+		},
+		{
+			Key:       "key2",
+			Kind:      snapshot.KindCommitted,
+			Size:      20,
+			Inodes:    200,
+			Timestamp: 123456,
+		},
+		{
+			Key:       "key3",
+			Kind:      snapshot.KindView,
+			Size:      0,
+			Inodes:    0,
+			Timestamp: 345678,
+		},
+	}
+	expected := &runtime.FilesystemUsage{
+		Timestamp:  123456,
+		FsId:       &runtime.FilesystemIdentifier{Mountpoint: testImageFSPath},
+		UsedBytes:  &runtime.UInt64Value{Value: 30},
+		InodesUsed: &runtime.UInt64Value{Value: 300},
+	}
+	for _, sn := range snapshots {
+		c.snapshotStore.Add(sn)
+	}
+	resp, err := c.ImageFsInfo(context.Background(), &runtime.ImageFsInfoRequest{})
+	require.NoError(t, err)
+	stats := resp.GetImageFilesystems()
+	assert.Len(t, stats, 1)
+	assert.Equal(t, expected, stats[0])
+}

pkg/cri/server/instrumented_service.go

@@ -0,0 +1,490 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"errors"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+)
+
+// instrumentedService wraps service with containerd namespace and logs.
+type instrumentedService struct {
+	c *criService
+}
+
+func newInstrumentedService(c *criService) grpcServices {
+	return &instrumentedService{c: c}
+}
+
+// checkInitialized returns error if the server is not fully initialized.
+// GRPC service request handlers should return error before server is fully
+// initialized.
+// NOTE(random-liu): All following functions MUST check initialized at the beginning.
+func (in *instrumentedService) checkInitialized() error {
+	if in.c.initialized.IsSet() {
+		return nil
+	}
+	return errors.New("server is not initialized yet")
+}
+
+func (in *instrumentedService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (res *runtime.RunPodSandboxResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("RunPodsandbox for %+v", r.GetConfig().GetMetadata())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("RunPodSandbox for %+v failed, error", r.GetConfig().GetMetadata())
+		} else {
+			log.G(ctx).Infof("RunPodSandbox for %+v returns sandbox id %q", r.GetConfig().GetMetadata(), res.GetPodSandboxId())
+		}
+	}()
+	res, err = in.c.RunPodSandbox(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ListPodSandbox(ctx context.Context, r *runtime.ListPodSandboxRequest) (res *runtime.ListPodSandboxResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ListPodSandbox with filter %+v", r.GetFilter())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("ListPodSandbox failed")
+		} else {
+			log.G(ctx).Tracef("ListPodSandbox returns pod sandboxes %+v", res.GetItems())
+		}
+	}()
+	res, err = in.c.ListPodSandbox(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandboxStatusRequest) (res *runtime.PodSandboxStatusResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("PodSandboxStatus for %q", r.GetPodSandboxId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("PodSandboxStatus for %q failed", r.GetPodSandboxId())
+		} else {
+			log.G(ctx).Tracef("PodSandboxStatus for %q returns status %+v", r.GetPodSandboxId(), res.GetStatus())
+		}
+	}()
+	res, err = in.c.PodSandboxStatus(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandboxRequest) (_ *runtime.StopPodSandboxResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("StopPodSandbox for %q", r.GetPodSandboxId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("StopPodSandbox for %q failed", r.GetPodSandboxId())
+		} else {
+			log.G(ctx).Infof("StopPodSandbox for %q returns successfully", r.GetPodSandboxId())
+		}
+	}()
+	res, err := in.c.StopPodSandbox(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodSandboxRequest) (_ *runtime.RemovePodSandboxResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("RemovePodSandbox for %q", r.GetPodSandboxId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("RemovePodSandbox for %q failed", r.GetPodSandboxId())
+		} else {
+			log.G(ctx).Infof("RemovePodSandbox %q returns successfully", r.GetPodSandboxId())
+		}
+	}()
+	res, err := in.c.RemovePodSandbox(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) PortForward(ctx context.Context, r *runtime.PortForwardRequest) (res *runtime.PortForwardResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("Portforward for %q port %v", r.GetPodSandboxId(), r.GetPort())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Portforward for %q failed", r.GetPodSandboxId())
+		} else {
+			log.G(ctx).Infof("Portforward for %q returns URL %q", r.GetPodSandboxId(), res.GetUrl())
+		}
+	}()
+	res, err = in.c.PortForward(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) CreateContainer(ctx context.Context, r *runtime.CreateContainerRequest) (res *runtime.CreateContainerResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("CreateContainer within sandbox %q for container %+v",
+		r.GetPodSandboxId(), r.GetConfig().GetMetadata())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("CreateContainer within sandbox %q for %+v failed",
+				r.GetPodSandboxId(), r.GetConfig().GetMetadata())
+		} else {
+			log.G(ctx).Infof("CreateContainer within sandbox %q for %+v returns container id %q",
+				r.GetPodSandboxId(), r.GetConfig().GetMetadata(), res.GetContainerId())
+		}
+	}()
+	res, err = in.c.CreateContainer(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (_ *runtime.StartContainerResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("StartContainer for %q", r.GetContainerId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("StartContainer for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("StartContainer for %q returns successfully", r.GetContainerId())
+		}
+	}()
+	res, err := in.c.StartContainer(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ListContainers(ctx context.Context, r *runtime.ListContainersRequest) (res *runtime.ListContainersResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ListContainers with filter %+v", r.GetFilter())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ListContainers with filter %+v failed", r.GetFilter())
+		} else {
+			log.G(ctx).Tracef("ListContainers with filter %+v returns containers %+v",
+				r.GetFilter(), res.GetContainers())
+		}
+	}()
+	res, err = in.c.ListContainers(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ContainerStatus(ctx context.Context, r *runtime.ContainerStatusRequest) (res *runtime.ContainerStatusResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ContainerStatus for %q", r.GetContainerId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ContainerStatus for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Tracef("ContainerStatus for %q returns status %+v", r.GetContainerId(), res.GetStatus())
+		}
+	}()
+	res, err = in.c.ContainerStatus(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) StopContainer(ctx context.Context, r *runtime.StopContainerRequest) (res *runtime.StopContainerResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("StopContainer for %q with timeout %d (s)", r.GetContainerId(), r.GetTimeout())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("StopContainer for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("StopContainer for %q returns successfully", r.GetContainerId())
+		}
+	}()
+	res, err = in.c.StopContainer(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) RemoveContainer(ctx context.Context, r *runtime.RemoveContainerRequest) (res *runtime.RemoveContainerResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("RemoveContainer for %q", r.GetContainerId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("RemoveContainer for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("RemoveContainer for %q returns successfully", r.GetContainerId())
+		}
+	}()
+	res, err = in.c.RemoveContainer(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ExecSync(ctx context.Context, r *runtime.ExecSyncRequest) (res *runtime.ExecSyncResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("ExecSync for %q with command %+v and timeout %d (s)", r.GetContainerId(), r.GetCmd(), r.GetTimeout())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ExecSync for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("ExecSync for %q returns with exit code %d", r.GetContainerId(), res.GetExitCode())
+			log.G(ctx).Debugf("ExecSync for %q outputs - stdout: %q, stderr: %q", r.GetContainerId(),
+				res.GetStdout(), res.GetStderr())
+		}
+	}()
+	res, err = in.c.ExecSync(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) Exec(ctx context.Context, r *runtime.ExecRequest) (res *runtime.ExecResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("Exec for %q with command %+v, tty %v and stdin %v",
+		r.GetContainerId(), r.GetCmd(), r.GetTty(), r.GetStdin())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Exec for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("Exec for %q returns URL %q", r.GetContainerId(), res.GetUrl())
+		}
+	}()
+	res, err = in.c.Exec(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) Attach(ctx context.Context, r *runtime.AttachRequest) (res *runtime.AttachResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("Attach for %q with tty %v and stdin %v", r.GetContainerId(), r.GetTty(), r.GetStdin())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Attach for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("Attach for %q returns URL %q", r.GetContainerId(), res.Url)
+		}
+	}()
+	res, err = in.c.Attach(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) UpdateContainerResources(ctx context.Context, r *runtime.UpdateContainerResourcesRequest) (res *runtime.UpdateContainerResourcesResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("UpdateContainerResources for %q with %+v", r.GetContainerId(), r.GetLinux())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("UpdateContainerResources for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Infof("UpdateContainerResources for %q returns successfully", r.GetContainerId())
+		}
+	}()
+	res, err = in.c.UpdateContainerResources(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (res *runtime.PullImageResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("PullImage %q", r.GetImage().GetImage())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("PullImage %q failed", r.GetImage().GetImage())
+		} else {
+			log.G(ctx).Infof("PullImage %q returns image reference %q",
+				r.GetImage().GetImage(), res.GetImageRef())
+		}
+	}()
+	res, err = in.c.PullImage(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ListImages(ctx context.Context, r *runtime.ListImagesRequest) (res *runtime.ListImagesResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ListImages with filter %+v", r.GetFilter())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ListImages with filter %+v failed", r.GetFilter())
+		} else {
+			log.G(ctx).Tracef("ListImages with filter %+v returns image list %+v",
+				r.GetFilter(), res.GetImages())
+		}
+	}()
+	res, err = in.c.ListImages(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (res *runtime.ImageStatusResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ImageStatus for %q", r.GetImage().GetImage())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ImageStatus for %q failed", r.GetImage().GetImage())
+		} else {
+			log.G(ctx).Tracef("ImageStatus for %q returns image status %+v",
+				r.GetImage().GetImage(), res.GetImage())
+		}
+	}()
+	res, err = in.c.ImageStatus(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) RemoveImage(ctx context.Context, r *runtime.RemoveImageRequest) (_ *runtime.RemoveImageResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Infof("RemoveImage %q", r.GetImage().GetImage())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("RemoveImage %q failed", r.GetImage().GetImage())
+		} else {
+			log.G(ctx).Infof("RemoveImage %q returns successfully", r.GetImage().GetImage())
+		}
+	}()
+	res, err := in.c.RemoveImage(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (res *runtime.ImageFsInfoResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Debugf("ImageFsInfo")
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("ImageFsInfo failed")
+		} else {
+			log.G(ctx).Debugf("ImageFsInfo returns filesystem info %+v", res.ImageFilesystems)
+		}
+	}()
+	res, err = in.c.ImageFsInfo(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ContainerStats(ctx context.Context, r *runtime.ContainerStatsRequest) (res *runtime.ContainerStatsResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Debugf("ContainerStats for %q", r.GetContainerId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ContainerStats for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Debugf("ContainerStats for %q returns stats %+v", r.GetContainerId(), res.GetStats())
+		}
+	}()
+	res, err = in.c.ContainerStats(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ListContainerStats(ctx context.Context, r *runtime.ListContainerStatsRequest) (res *runtime.ListContainerStatsResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("ListContainerStats with filter %+v", r.GetFilter())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("ListContainerStats failed")
+		} else {
+			log.G(ctx).Tracef("ListContainerStats returns stats %+v", res.GetStats())
+		}
+	}()
+	res, err = in.c.ListContainerStats(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) Status(ctx context.Context, r *runtime.StatusRequest) (res *runtime.StatusResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("Status")
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("Status failed")
+		} else {
+			log.G(ctx).Tracef("Status returns status %+v", res.GetStatus())
+		}
+	}()
+	res, err = in.c.Status(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) Version(ctx context.Context, r *runtime.VersionRequest) (res *runtime.VersionResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("Version with client side version %q", r.GetVersion())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("Version failed")
+		} else {
+			log.G(ctx).Tracef("Version returns %+v", res)
+		}
+	}()
+	res, err = in.c.Version(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (res *runtime.UpdateRuntimeConfigResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Debugf("UpdateRuntimeConfig with config %+v", r.GetRuntimeConfig())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("UpdateRuntimeConfig failed")
+		} else {
+			log.G(ctx).Debug("UpdateRuntimeConfig returns returns successfully")
+		}
+	}()
+	res, err = in.c.UpdateRuntimeConfig(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}
+
+func (in *instrumentedService) ReopenContainerLog(ctx context.Context, r *runtime.ReopenContainerLogRequest) (res *runtime.ReopenContainerLogResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Debugf("ReopenContainerLog for %q", r.GetContainerId())
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("ReopenContainerLog for %q failed", r.GetContainerId())
+		} else {
+			log.G(ctx).Debugf("ReopenContainerLog for %q returns successfully", r.GetContainerId())
+		}
+	}()
+	res, err = in.c.ReopenContainerLog(ctrdutil.WithNamespace(ctx), r)
+	return res, errdefs.ToGRPC(err)
+}

pkg/cri/server/opts.go

@@ -0,0 +1,51 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+)
+
+// WithNRISandboxDelete calls delete for a sandbox'd task
+func WithNRISandboxDelete(sandboxID string) containerd.ProcessDeleteOpts {
+	return func(ctx context.Context, p containerd.Process) error {
+		task, ok := p.(containerd.Task)
+		if !ok {
+			return nil
+		}
+		nric, err := nri.New()
+		if err != nil {
+			log.G(ctx).WithError(err).Error("unable to create nri client")
+			return nil
+		}
+		if nric == nil {
+			return nil
+		}
+		sb := &nri.Sandbox{
+			ID: sandboxID,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Delete, sb); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to delete nri for %q", task.ID())
+		}
+		return nil
+	}
+}

pkg/cri/server/restart.go

@@ -0,0 +1,483 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	goruntime "runtime"
+	"time"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	cio "github.com/containerd/containerd/pkg/cri/io"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	"github.com/containerd/containerd/pkg/netns"
+)
+
+// NOTE: The recovery logic has following assumption: when the cri plugin is down:
+// 1) Files (e.g. root directory, netns) and checkpoint maintained by the plugin MUST NOT be
+// touched. Or else, recovery logic for those containers/sandboxes may return error.
+// 2) Containerd containers may be deleted, but SHOULD NOT be added. Or else, recovery logic
+// for the newly added container/sandbox will return error, because there is no corresponding root
+// directory created.
+// 3) Containerd container tasks may exit or be stoppped, deleted. Even though current logic could
+// tolerant tasks being created or started, we prefer that not to happen.
+
+// recover recovers system state from containerd and status checkpoint.
+func (c *criService) recover(ctx context.Context) error {
+	// Recover all sandboxes.
+	sandboxes, err := c.client.Containers(ctx, filterLabel(containerKindLabel, containerKindSandbox))
+	if err != nil {
+		return errors.Wrap(err, "failed to list sandbox containers")
+	}
+	for _, sandbox := range sandboxes {
+		sb, err := c.loadSandbox(ctx, sandbox)
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to load sandbox %q", sandbox.ID())
+			continue
+		}
+		log.G(ctx).Debugf("Loaded sandbox %+v", sb)
+		if err := c.sandboxStore.Add(sb); err != nil {
+			return errors.Wrapf(err, "failed to add sandbox %q to store", sandbox.ID())
+		}
+		if err := c.sandboxNameIndex.Reserve(sb.Name, sb.ID); err != nil {
+			return errors.Wrapf(err, "failed to reserve sandbox name %q", sb.Name)
+		}
+	}
+
+	// Recover all containers.
+	containers, err := c.client.Containers(ctx, filterLabel(containerKindLabel, containerKindContainer))
+	if err != nil {
+		return errors.Wrap(err, "failed to list containers")
+	}
+	for _, container := range containers {
+		cntr, err := c.loadContainer(ctx, container)
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to load container %q", container.ID())
+			continue
+		}
+		log.G(ctx).Debugf("Loaded container %+v", cntr)
+		if err := c.containerStore.Add(cntr); err != nil {
+			return errors.Wrapf(err, "failed to add container %q to store", container.ID())
+		}
+		if err := c.containerNameIndex.Reserve(cntr.Name, cntr.ID); err != nil {
+			return errors.Wrapf(err, "failed to reserve container name %q", cntr.Name)
+		}
+	}
+
+	// Recover all images.
+	cImages, err := c.client.ListImages(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to list images")
+	}
+	c.loadImages(ctx, cImages)
+
+	// It's possible that containerd containers are deleted unexpectedly. In that case,
+	// we can't even get metadata, we should cleanup orphaned sandbox/container directories
+	// with best effort.
+
+	// Cleanup orphaned sandbox and container directories without corresponding containerd container.
+	for _, cleanup := range []struct {
+		cntrs  []containerd.Container
+		base   string
+		errMsg string
+	}{
+		{
+			cntrs:  sandboxes,
+			base:   filepath.Join(c.config.RootDir, sandboxesDir),
+			errMsg: "failed to cleanup orphaned sandbox directories",
+		},
+		{
+			cntrs:  sandboxes,
+			base:   filepath.Join(c.config.StateDir, sandboxesDir),
+			errMsg: "failed to cleanup orphaned volatile sandbox directories",
+		},
+		{
+			cntrs:  containers,
+			base:   filepath.Join(c.config.RootDir, containersDir),
+			errMsg: "failed to cleanup orphaned container directories",
+		},
+		{
+			cntrs:  containers,
+			base:   filepath.Join(c.config.StateDir, containersDir),
+			errMsg: "failed to cleanup orphaned volatile container directories",
+		},
+	} {
+		if err := cleanupOrphanedIDDirs(ctx, cleanup.cntrs, cleanup.base); err != nil {
+			return errors.Wrap(err, cleanup.errMsg)
+		}
+	}
+	return nil
+}
+
+// loadContainerTimeout is the default timeout for loading a container/sandbox.
+// One container/sandbox hangs (e.g. containerd#2438) should not affect other
+// containers/sandboxes.
+// Most CRI container/sandbox related operations are per container, the ones
+// which handle multiple containers at a time are:
+// * ListPodSandboxes: Don't talk with containerd services.
+// * ListContainers: Don't talk with containerd services.
+// * ListContainerStats: Not in critical code path, a default timeout will
+// be applied at CRI level.
+// * Recovery logic: We should set a time for each container/sandbox recovery.
+// * Event monitor: We should set a timeout for each container/sandbox event handling.
+const loadContainerTimeout = 10 * time.Second
+
+// loadContainer loads container from containerd and status checkpoint.
+func (c *criService) loadContainer(ctx context.Context, cntr containerd.Container) (containerstore.Container, error) {
+	ctx, cancel := context.WithTimeout(ctx, loadContainerTimeout)
+	defer cancel()
+	id := cntr.ID()
+	containerDir := c.getContainerRootDir(id)
+	volatileContainerDir := c.getVolatileContainerRootDir(id)
+	var container containerstore.Container
+	// Load container metadata.
+	exts, err := cntr.Extensions(ctx)
+	if err != nil {
+		return container, errors.Wrap(err, "failed to get container extensions")
+	}
+	ext, ok := exts[containerMetadataExtension]
+	if !ok {
+		return container, errors.Errorf("metadata extension %q not found", containerMetadataExtension)
+	}
+	data, err := typeurl.UnmarshalAny(&ext)
+	if err != nil {
+		return container, errors.Wrapf(err, "failed to unmarshal metadata extension %q", ext)
+	}
+	meta := data.(*containerstore.Metadata)
+
+	// Load status from checkpoint.
+	status, err := containerstore.LoadStatus(containerDir, id)
+	if err != nil {
+		log.G(ctx).WithError(err).Warnf("Failed to load container status for %q", id)
+		status = unknownContainerStatus()
+	}
+
+	var containerIO *cio.ContainerIO
+	err = func() error {
+		// Load up-to-date status from containerd.
+		t, err := cntr.Task(ctx, func(fifos *containerdio.FIFOSet) (_ containerdio.IO, err error) {
+			stdoutWC, stderrWC, err := c.createContainerLoggers(meta.LogPath, meta.Config.GetTty())
+			if err != nil {
+				return nil, err
+			}
+			defer func() {
+				if err != nil {
+					if stdoutWC != nil {
+						stdoutWC.Close()
+					}
+					if stderrWC != nil {
+						stderrWC.Close()
+					}
+				}
+			}()
+			containerIO, err = cio.NewContainerIO(id,
+				cio.WithFIFOs(fifos),
+			)
+			if err != nil {
+				return nil, err
+			}
+			containerIO.AddOutput("log", stdoutWC, stderrWC)
+			containerIO.Pipe()
+			return containerIO, nil
+		})
+		if err != nil && !errdefs.IsNotFound(err) {
+			return errors.Wrap(err, "failed to load task")
+		}
+		var s containerd.Status
+		var notFound bool
+		if errdefs.IsNotFound(err) {
+			// Task is not found.
+			notFound = true
+		} else {
+			// Task is found. Get task status.
+			s, err = t.Status(ctx)
+			if err != nil {
+				// It's still possible that task is deleted during this window.
+				if !errdefs.IsNotFound(err) {
+					return errors.Wrap(err, "failed to get task status")
+				}
+				notFound = true
+			}
+		}
+		if notFound {
+			// Task is not created or has been deleted, use the checkpointed status
+			// to generate container status.
+			switch status.State() {
+			case runtime.ContainerState_CONTAINER_CREATED:
+				// NOTE: Another possibility is that we've tried to start the container, but
+				// containerd got restarted during that. In that case, we still
+				// treat the container as `CREATED`.
+				containerIO, err = cio.NewContainerIO(id,
+					cio.WithNewFIFOs(volatileContainerDir, meta.Config.GetTty(), meta.Config.GetStdin()),
+				)
+				if err != nil {
+					return errors.Wrap(err, "failed to create container io")
+				}
+			case runtime.ContainerState_CONTAINER_RUNNING:
+				// Container was in running state, but its task has been deleted,
+				// set unknown exited state. Container io is not needed in this case.
+				status.FinishedAt = time.Now().UnixNano()
+				status.ExitCode = unknownExitCode
+				status.Reason = unknownExitReason
+			default:
+				// Container is in exited/unknown state, return the status as it is.
+			}
+		} else {
+			// Task status is found. Update container status based on the up-to-date task status.
+			switch s.Status {
+			case containerd.Created:
+				// Task has been created, but not started yet. This could only happen if containerd
+				// gets restarted during container start.
+				// Container must be in `CREATED` state.
+				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+					return errors.Wrap(err, "failed to delete task")
+				}
+				if status.State() != runtime.ContainerState_CONTAINER_CREATED {
+					return errors.Errorf("unexpected container state for created task: %q", status.State())
+				}
+			case containerd.Running:
+				// Task is running. Container must be in `RUNNING` state, based on our assuption that
+				// "task should not be started when containerd is down".
+				switch status.State() {
+				case runtime.ContainerState_CONTAINER_EXITED:
+					return errors.Errorf("unexpected container state for running task: %q", status.State())
+				case runtime.ContainerState_CONTAINER_RUNNING:
+				default:
+					// This may happen if containerd gets restarted after task is started, but
+					// before status is checkpointed.
+					status.StartedAt = time.Now().UnixNano()
+					status.Pid = t.Pid()
+				}
+				// Wait for the task for exit monitor.
+				// wait is a long running background request, no timeout needed.
+				exitCh, err := t.Wait(ctrdutil.NamespacedContext())
+				if err != nil {
+					if !errdefs.IsNotFound(err) {
+						return errors.Wrap(err, "failed to wait for task")
+					}
+					// Container was in running state, but its task has been deleted,
+					// set unknown exited state.
+					status.FinishedAt = time.Now().UnixNano()
+					status.ExitCode = unknownExitCode
+					status.Reason = unknownExitReason
+				} else {
+					// Start exit monitor.
+					c.eventMonitor.startExitMonitor(context.Background(), id, status.Pid, exitCh)
+				}
+			case containerd.Stopped:
+				// Task is stopped. Updata status and delete the task.
+				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+					return errors.Wrap(err, "failed to delete task")
+				}
+				status.FinishedAt = s.ExitTime.UnixNano()
+				status.ExitCode = int32(s.ExitStatus)
+			default:
+				return errors.Errorf("unexpected task status %q", s.Status)
+			}
+		}
+		return nil
+	}()
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("Failed to load container status for %q", id)
+		// Only set the unknown field in this case, because other fields may
+		// contain useful information loaded from the checkpoint.
+		status.Unknown = true
+	}
+	opts := []containerstore.Opts{
+		containerstore.WithStatus(status, containerDir),
+		containerstore.WithContainer(cntr),
+	}
+	// containerIO could be nil for container in unknown state.
+	if containerIO != nil {
+		opts = append(opts, containerstore.WithContainerIO(containerIO))
+	}
+	return containerstore.NewContainer(*meta, opts...)
+}
+
+// loadSandbox loads sandbox from containerd.
+func (c *criService) loadSandbox(ctx context.Context, cntr containerd.Container) (sandboxstore.Sandbox, error) {
+	ctx, cancel := context.WithTimeout(ctx, loadContainerTimeout)
+	defer cancel()
+	var sandbox sandboxstore.Sandbox
+	// Load sandbox metadata.
+	exts, err := cntr.Extensions(ctx)
+	if err != nil {
+		return sandbox, errors.Wrap(err, "failed to get sandbox container extensions")
+	}
+	ext, ok := exts[sandboxMetadataExtension]
+	if !ok {
+		return sandbox, errors.Errorf("metadata extension %q not found", sandboxMetadataExtension)
+	}
+	data, err := typeurl.UnmarshalAny(&ext)
+	if err != nil {
+		return sandbox, errors.Wrapf(err, "failed to unmarshal metadata extension %q", ext)
+	}
+	meta := data.(*sandboxstore.Metadata)
+
+	s, err := func() (sandboxstore.Status, error) {
+		status := unknownSandboxStatus()
+		// Load sandbox created timestamp.
+		info, err := cntr.Info(ctx)
+		if err != nil {
+			return status, errors.Wrap(err, "failed to get sandbox container info")
+		}
+		status.CreatedAt = info.CreatedAt
+
+		// Load sandbox state.
+		t, err := cntr.Task(ctx, nil)
+		if err != nil && !errdefs.IsNotFound(err) {
+			return status, errors.Wrap(err, "failed to load task")
+		}
+		var taskStatus containerd.Status
+		var notFound bool
+		if errdefs.IsNotFound(err) {
+			// Task is not found.
+			notFound = true
+		} else {
+			// Task is found. Get task status.
+			taskStatus, err = t.Status(ctx)
+			if err != nil {
+				// It's still possible that task is deleted during this window.
+				if !errdefs.IsNotFound(err) {
+					return status, errors.Wrap(err, "failed to get task status")
+				}
+				notFound = true
+			}
+		}
+		if notFound {
+			// Task does not exist, set sandbox state as NOTREADY.
+			status.State = sandboxstore.StateNotReady
+		} else {
+			if taskStatus.Status == containerd.Running {
+				// Wait for the task for sandbox monitor.
+				// wait is a long running background request, no timeout needed.
+				exitCh, err := t.Wait(ctrdutil.NamespacedContext())
+				if err != nil {
+					if !errdefs.IsNotFound(err) {
+						return status, errors.Wrap(err, "failed to wait for task")
+					}
+					status.State = sandboxstore.StateNotReady
+				} else {
+					// Task is running, set sandbox state as READY.
+					status.State = sandboxstore.StateReady
+					status.Pid = t.Pid()
+					c.eventMonitor.startExitMonitor(context.Background(), meta.ID, status.Pid, exitCh)
+				}
+			} else {
+				// Task is not running. Delete the task and set sandbox state as NOTREADY.
+				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+					return status, errors.Wrap(err, "failed to delete task")
+				}
+				status.State = sandboxstore.StateNotReady
+			}
+		}
+		return status, nil
+	}()
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("Failed to load sandbox status for %q", cntr.ID())
+	}
+
+	sandbox = sandboxstore.NewSandbox(*meta, s)
+	sandbox.Container = cntr
+
+	// Load network namespace.
+	if goruntime.GOOS != "windows" &&
+		meta.Config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtime.NamespaceMode_NODE {
+		// Don't need to load netns for host network sandbox.
+		return sandbox, nil
+	}
+	sandbox.NetNS = netns.LoadNetNS(meta.NetNSPath)
+
+	// It doesn't matter whether task is running or not. If it is running, sandbox
+	// status will be `READY`; if it is not running, sandbox status will be `NOT_READY`,
+	// kubelet will stop the sandbox which will properly cleanup everything.
+	return sandbox, nil
+}
+
+// loadImages loads images from containerd.
+func (c *criService) loadImages(ctx context.Context, cImages []containerd.Image) {
+	snapshotter := c.config.ContainerdConfig.Snapshotter
+	for _, i := range cImages {
+		ok, _, _, _, err := containerdimages.Check(ctx, i.ContentStore(), i.Target(), platforms.Default())
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to check image content readiness for %q", i.Name())
+			continue
+		}
+		if !ok {
+			log.G(ctx).Warnf("The image content readiness for %q is not ok", i.Name())
+			continue
+		}
+		// Checking existence of top-level snapshot for each image being recovered.
+		unpacked, err := i.IsUnpacked(ctx, snapshotter)
+		if err != nil {
+			log.G(ctx).WithError(err).Warnf("Failed to check whether image is unpacked for image %s", i.Name())
+			continue
+		}
+		if !unpacked {
+			log.G(ctx).Warnf("The image %s is not unpacked.", i.Name())
+			// TODO(random-liu): Consider whether we should try unpack here.
+		}
+		if err := c.updateImage(ctx, i.Name()); err != nil {
+			log.G(ctx).WithError(err).Warnf("Failed to update reference for image %q", i.Name())
+			continue
+		}
+		log.G(ctx).Debugf("Loaded image %q", i.Name())
+	}
+}
+
+func cleanupOrphanedIDDirs(ctx context.Context, cntrs []containerd.Container, base string) error {
+	// Cleanup orphaned id directories.
+	dirs, err := ioutil.ReadDir(base)
+	if err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "failed to read base directory")
+	}
+	idsMap := make(map[string]containerd.Container)
+	for _, cntr := range cntrs {
+		idsMap[cntr.ID()] = cntr
+	}
+	for _, d := range dirs {
+		if !d.IsDir() {
+			log.G(ctx).Warnf("Invalid file %q found in base directory %q", d.Name(), base)
+			continue
+		}
+		if _, ok := idsMap[d.Name()]; ok {
+			// Do not remove id directory if corresponding container is found.
+			continue
+		}
+		dir := filepath.Join(base, d.Name())
+		if err := ensureRemoveAll(ctx, dir); err != nil {
+			log.G(ctx).WithError(err).Warnf("Failed to remove id directory %q", dir)
+		} else {
+			log.G(ctx).Debugf("Cleanup orphaned id directory %q", dir)
+		}
+	}
+	return nil
+}

pkg/cri/server/sandbox_list.go

@@ -0,0 +1,101 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+// ListPodSandbox returns a list of Sandbox.
+func (c *criService) ListPodSandbox(ctx context.Context, r *runtime.ListPodSandboxRequest) (*runtime.ListPodSandboxResponse, error) {
+	// List all sandboxes from store.
+	sandboxesInStore := c.sandboxStore.List()
+	var sandboxes []*runtime.PodSandbox
+	for _, sandboxInStore := range sandboxesInStore {
+		sandboxes = append(sandboxes, toCRISandbox(
+			sandboxInStore.Metadata,
+			sandboxInStore.Status.Get(),
+		))
+	}
+
+	sandboxes = c.filterCRISandboxes(sandboxes, r.GetFilter())
+	return &runtime.ListPodSandboxResponse{Items: sandboxes}, nil
+}
+
+// toCRISandbox converts sandbox metadata into CRI pod sandbox.
+func toCRISandbox(meta sandboxstore.Metadata, status sandboxstore.Status) *runtime.PodSandbox {
+	// Set sandbox state to NOTREADY by default.
+	state := runtime.PodSandboxState_SANDBOX_NOTREADY
+	if status.State == sandboxstore.StateReady {
+		state = runtime.PodSandboxState_SANDBOX_READY
+	}
+	return &runtime.PodSandbox{
+		Id:             meta.ID,
+		Metadata:       meta.Config.GetMetadata(),
+		State:          state,
+		CreatedAt:      status.CreatedAt.UnixNano(),
+		Labels:         meta.Config.GetLabels(),
+		Annotations:    meta.Config.GetAnnotations(),
+		RuntimeHandler: meta.RuntimeHandler,
+	}
+}
+
+func (c *criService) normalizePodSandboxFilter(filter *runtime.PodSandboxFilter) {
+	if sb, err := c.sandboxStore.Get(filter.GetId()); err == nil {
+		filter.Id = sb.ID
+	}
+}
+
+// filterCRISandboxes filters CRISandboxes.
+func (c *criService) filterCRISandboxes(sandboxes []*runtime.PodSandbox, filter *runtime.PodSandboxFilter) []*runtime.PodSandbox {
+	if filter == nil {
+		return sandboxes
+	}
+
+	c.normalizePodSandboxFilter(filter)
+	filtered := []*runtime.PodSandbox{}
+	for _, s := range sandboxes {
+		// Filter by id
+		if filter.GetId() != "" && filter.GetId() != s.Id {
+			continue
+		}
+		// Filter by state
+		if filter.GetState() != nil && filter.GetState().GetState() != s.State {
+			continue
+		}
+		// Filter by label
+		if filter.GetLabelSelector() != nil {
+			match := true
+			for k, v := range filter.GetLabelSelector() {
+				got, ok := s.Labels[k]
+				if !ok || got != v {
+					match = false
+					break
+				}
+			}
+			if !match {
+				continue
+			}
+		}
+		filtered = append(filtered, s)
+	}
+
+	return filtered
+}

pkg/cri/server/sandbox_list_test.go

@@ -0,0 +1,208 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestToCRISandbox(t *testing.T) {
+	config := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-name",
+			Uid:       "test-uid",
+			Namespace: "test-ns",
+			Attempt:   1,
+		},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+	createdAt := time.Now()
+	meta := sandboxstore.Metadata{
+		ID:             "test-id",
+		Name:           "test-name",
+		Config:         config,
+		NetNSPath:      "test-netns",
+		RuntimeHandler: "test-runtime-handler",
+	}
+	expect := &runtime.PodSandbox{
+		Id:             "test-id",
+		Metadata:       config.GetMetadata(),
+		CreatedAt:      createdAt.UnixNano(),
+		Labels:         config.GetLabels(),
+		Annotations:    config.GetAnnotations(),
+		RuntimeHandler: "test-runtime-handler",
+	}
+	for desc, test := range map[string]struct {
+		state         sandboxstore.State
+		expectedState runtime.PodSandboxState
+	}{
+		"sandbox state ready": {
+			state:         sandboxstore.StateReady,
+			expectedState: runtime.PodSandboxState_SANDBOX_READY,
+		},
+		"sandbox state not ready": {
+			state:         sandboxstore.StateNotReady,
+			expectedState: runtime.PodSandboxState_SANDBOX_NOTREADY,
+		},
+		"sandbox state unknown": {
+			state:         sandboxstore.StateUnknown,
+			expectedState: runtime.PodSandboxState_SANDBOX_NOTREADY,
+		},
+	} {
+		status := sandboxstore.Status{
+			CreatedAt: createdAt,
+			State:     test.state,
+		}
+		expect.State = test.expectedState
+		s := toCRISandbox(meta, status)
+		assert.Equal(t, expect, s, desc)
+	}
+}
+
+func TestFilterSandboxes(t *testing.T) {
+	c := newTestCRIService()
+	sandboxes := []sandboxstore.Sandbox{
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:   "1abcdef",
+				Name: "sandboxname-1",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "podname-1",
+						Uid:       "uid-1",
+						Namespace: "ns-1",
+						Attempt:   1,
+					},
+				},
+				RuntimeHandler: "test-runtime-handler",
+			},
+			sandboxstore.Status{
+				CreatedAt: time.Now(),
+				State:     sandboxstore.StateReady,
+			},
+		),
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:   "2abcdef",
+				Name: "sandboxname-2",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "podname-2",
+						Uid:       "uid-2",
+						Namespace: "ns-2",
+						Attempt:   2,
+					},
+					Labels: map[string]string{"a": "b"},
+				},
+				RuntimeHandler: "test-runtime-handler",
+			},
+			sandboxstore.Status{
+				CreatedAt: time.Now(),
+				State:     sandboxstore.StateNotReady,
+			},
+		),
+		sandboxstore.NewSandbox(
+			sandboxstore.Metadata{
+				ID:   "3abcdef",
+				Name: "sandboxname-3",
+				Config: &runtime.PodSandboxConfig{
+					Metadata: &runtime.PodSandboxMetadata{
+						Name:      "podname-2",
+						Uid:       "uid-2",
+						Namespace: "ns-2",
+						Attempt:   2,
+					},
+					Labels: map[string]string{"c": "d"},
+				},
+				RuntimeHandler: "test-runtime-handler",
+			},
+			sandboxstore.Status{
+				CreatedAt: time.Now(),
+				State:     sandboxstore.StateReady,
+			},
+		),
+	}
+
+	// Create PodSandbox
+	testSandboxes := []*runtime.PodSandbox{}
+	for _, sb := range sandboxes {
+		testSandboxes = append(testSandboxes, toCRISandbox(sb.Metadata, sb.Status.Get()))
+	}
+
+	// Inject test sandbox metadata
+	for _, sb := range sandboxes {
+		assert.NoError(t, c.sandboxStore.Add(sb))
+	}
+
+	for desc, test := range map[string]struct {
+		filter *runtime.PodSandboxFilter
+		expect []*runtime.PodSandbox
+	}{
+		"no filter": {
+			expect: testSandboxes,
+		},
+		"id filter": {
+			filter: &runtime.PodSandboxFilter{Id: "2abcdef"},
+			expect: []*runtime.PodSandbox{testSandboxes[1]},
+		},
+		"truncid filter": {
+			filter: &runtime.PodSandboxFilter{Id: "2"},
+			expect: []*runtime.PodSandbox{testSandboxes[1]},
+		},
+		"state filter": {
+			filter: &runtime.PodSandboxFilter{
+				State: &runtime.PodSandboxStateValue{
+					State: runtime.PodSandboxState_SANDBOX_READY,
+				},
+			},
+			expect: []*runtime.PodSandbox{testSandboxes[0], testSandboxes[2]},
+		},
+		"label filter": {
+			filter: &runtime.PodSandboxFilter{
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.PodSandbox{testSandboxes[1]},
+		},
+		"mixed filter not matched": {
+			filter: &runtime.PodSandboxFilter{
+				Id:            "1",
+				LabelSelector: map[string]string{"a": "b"},
+			},
+			expect: []*runtime.PodSandbox{},
+		},
+		"mixed filter matched": {
+			filter: &runtime.PodSandboxFilter{
+				State: &runtime.PodSandboxStateValue{
+					State: runtime.PodSandboxState_SANDBOX_READY,
+				},
+				LabelSelector: map[string]string{"c": "d"},
+			},
+			expect: []*runtime.PodSandbox{testSandboxes[2]},
+		},
+	} {
+		t.Logf("TestCase: %s", desc)
+		filtered := c.filterCRISandboxes(testSandboxes, test.filter)
+		assert.Equal(t, test.expect, filtered, desc)
+	}
+}

pkg/cri/server/sandbox_portforward.go

@@ -0,0 +1,38 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+// PortForward prepares a streaming endpoint to forward ports from a PodSandbox, and returns the address.
+func (c *criService) PortForward(ctx context.Context, r *runtime.PortForwardRequest) (retRes *runtime.PortForwardResponse, retErr error) {
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find sandbox %q", r.GetPodSandboxId())
+	}
+	if sandbox.Status.Get().State != sandboxstore.StateReady {
+		return nil, errors.New("sandbox container is not running")
+	}
+	// TODO(random-liu): Verify that ports are exposed.
+	return c.streamServer.GetPortForward(r)
+}

pkg/cri/server/sandbox_portforward_linux.go

@@ -0,0 +1,126 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/containerd/containerd/log"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// portForward uses netns to enter the sandbox namespace, and forwards a stream inside the
+// the namespace to a specific port. It keeps forwarding until it exits or client disconnect.
+func (c *criService) portForward(ctx context.Context, id string, port int32, stream io.ReadWriteCloser) error {
+	s, err := c.sandboxStore.Get(id)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find sandbox %q in store", id)
+	}
+
+	var netNSDo func(func(ns.NetNS) error) error
+	// netNSPath is the network namespace path for logging.
+	var netNSPath string
+	securityContext := s.Config.GetLinux().GetSecurityContext()
+	hostNet := securityContext.GetNamespaceOptions().GetNetwork() == runtime.NamespaceMode_NODE
+	if !hostNet {
+		if closed, err := s.NetNS.Closed(); err != nil {
+			return errors.Wrapf(err, "failed to check netwok namespace closed for sandbox %q", id)
+		} else if closed {
+			return errors.Errorf("network namespace for sandbox %q is closed", id)
+		}
+		netNSDo = s.NetNS.Do
+		netNSPath = s.NetNS.GetPath()
+	} else {
+		// Run the function directly for host network.
+		netNSDo = func(do func(_ ns.NetNS) error) error {
+			return do(nil)
+		}
+		netNSPath = "host"
+	}
+
+	log.G(ctx).Infof("Executing port forwarding in network namespace %q", netNSPath)
+	err = netNSDo(func(_ ns.NetNS) error {
+		defer stream.Close()
+		// TODO: hardcoded to tcp4 because localhost resolves to ::1 by default if the system has IPv6 enabled.
+		// Theoretically happy eyeballs will try IPv6 first and fallback to IPv4
+		// but resolving localhost doesn't seem to return and IPv4 address, thus failing the connection.
+		conn, err := net.Dial("tcp4", fmt.Sprintf("localhost:%d", port))
+		if err != nil {
+			return errors.Wrapf(err, "failed to dial %d", port)
+		}
+		defer conn.Close()
+
+		errCh := make(chan error, 2)
+		// Copy from the the namespace port connection to the client stream
+		go func() {
+			log.G(ctx).Debugf("PortForward copying data from namespace %q port %d to the client stream", id, port)
+			_, err := io.Copy(stream, conn)
+			errCh <- err
+		}()
+
+		// Copy from the client stream to the namespace port connection
+		go func() {
+			log.G(ctx).Debugf("PortForward copying data from client stream to namespace %q port %d", id, port)
+			_, err := io.Copy(conn, stream)
+			errCh <- err
+		}()
+
+		// Wait until the first error is returned by one of the connections
+		// we use errFwd to store the result of the port forwarding operation
+		// if the context is cancelled close everything and return
+		var errFwd error
+		select {
+		case errFwd = <-errCh:
+			log.G(ctx).Debugf("PortForward stop forwarding in one direction in network namespace %q port %d: %v", id, port, errFwd)
+		case <-ctx.Done():
+			log.G(ctx).Debugf("PortForward cancelled in network namespace %q port %d: %v", id, port, ctx.Err())
+			return ctx.Err()
+		}
+		// give a chance to terminate gracefully or timeout
+		// after 1s
+		// https://linux.die.net/man/1/socat
+		const timeout = time.Second
+		select {
+		case e := <-errCh:
+			if errFwd == nil {
+				errFwd = e
+			}
+			log.G(ctx).Debugf("PortForward stopped forwarding in both directions in network namespace %q port %d: %v", id, port, e)
+		case <-time.After(timeout):
+			log.G(ctx).Debugf("PortForward timed out waiting to close the connection in network namespace %q port %d", id, port)
+		case <-ctx.Done():
+			log.G(ctx).Debugf("PortForward cancelled in network namespace %q port %d: %v", id, port, ctx.Err())
+			errFwd = ctx.Err()
+		}
+
+		return errFwd
+	})
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to execute portforward in network namespace %q", netNSPath)
+	}
+	log.G(ctx).Infof("Finish port forwarding for %q port %d", id, port)
+
+	return nil
+}

pkg/cri/server/sandbox_portforward_other.go

@@ -0,0 +1,33 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+// portForward uses netns to enter the sandbox namespace, and forwards a stream inside the
+// the namespace to a specific port. It keeps forwarding until it exits or client disconnect.
+func (c *criService) portForward(ctx context.Context, id string, port int32, stream io.ReadWriteCloser) error {
+	return errors.Wrap(errdefs.ErrNotImplemented, "port forward")
+}

pkg/cri/server/sandbox_portforward_windows.go

@@ -0,0 +1,80 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"k8s.io/utils/exec"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	"github.com/containerd/containerd/pkg/ioutil"
+)
+
+func (c *criService) portForward(ctx context.Context, id string, port int32, stream io.ReadWriter) error {
+	stdout := ioutil.NewNopWriteCloser(stream)
+	stderrBuffer := new(bytes.Buffer)
+	stderr := ioutil.NewNopWriteCloser(stderrBuffer)
+	// localhost is resolved to 127.0.0.1 in ipv4, and ::1 in ipv6.
+	// Explicitly using ipv4 IP address in here to avoid flakiness.
+	cmd := []string{"wincat.exe", "127.0.0.1", fmt.Sprint(port)}
+	err := c.execInSandbox(ctx, id, cmd, stream, stdout, stderr)
+	if err != nil {
+		return errors.Wrapf(err, "failed to execute port forward in sandbox: %s", stderrBuffer.String())
+	}
+	return nil
+}
+
+func (c *criService) execInSandbox(ctx context.Context, sandboxID string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser) error {
+	// Get sandbox from our sandbox store.
+	sb, err := c.sandboxStore.Get(sandboxID)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find sandbox %q in store", sandboxID)
+	}
+
+	// Check the sandbox state
+	state := sb.Status.Get().State
+	if state != sandboxstore.StateReady {
+		return errors.Errorf("sandbox is in %s state", fmt.Sprint(state))
+	}
+
+	opts := execOptions{
+		cmd:    cmd,
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+		tty:    false,
+		resize: nil,
+	}
+	exitCode, err := c.execInternal(ctx, sb.Container, sandboxID, opts)
+	if err != nil {
+		return errors.Wrap(err, "failed to exec in sandbox")
+	}
+	if *exitCode == 0 {
+		return nil
+	}
+	return &exec.CodeExitError{
+		Err:  errors.Errorf("error executing command %v, exit code %d", cmd, *exitCode),
+		Code: int(*exitCode),
+	}
+}

pkg/cri/server/sandbox_remove.go

@@ -0,0 +1,115 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+// RemovePodSandbox removes the sandbox. If there are running containers in the
+// sandbox, they should be forcibly removed.
+func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodSandboxRequest) (*runtime.RemovePodSandboxResponse, error) {
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		if err != store.ErrNotExist {
+			return nil, errors.Wrapf(err, "an error occurred when try to find sandbox %q",
+				r.GetPodSandboxId())
+		}
+		// Do not return error if the id doesn't exist.
+		log.G(ctx).Tracef("RemovePodSandbox called for sandbox %q that does not exist",
+			r.GetPodSandboxId())
+		return &runtime.RemovePodSandboxResponse{}, nil
+	}
+	// Use the full sandbox id.
+	id := sandbox.ID
+
+	// Return error if sandbox container is still running or unknown.
+	state := sandbox.Status.Get().State
+	if state == sandboxstore.StateReady || state == sandboxstore.StateUnknown {
+		logrus.Infof("Forcibly stopping sandbox %q", id)
+		if err := c.stopPodSandbox(ctx, sandbox); err != nil {
+			return nil, errors.Wrapf(err, "failed to forcibly stop sandbox %q", id)
+		}
+	}
+
+	// Return error if sandbox network namespace is not closed yet.
+	if sandbox.NetNS != nil {
+		nsPath := sandbox.NetNS.GetPath()
+		if closed, err := sandbox.NetNS.Closed(); err != nil {
+			return nil, errors.Wrapf(err, "failed to check sandbox network namespace %q closed", nsPath)
+		} else if !closed {
+			return nil, errors.Errorf("sandbox network namespace %q is not fully closed", nsPath)
+		}
+	}
+
+	// Remove all containers inside the sandbox.
+	// NOTE(random-liu): container could still be created after this point, Kubelet should
+	// not rely on this behavior.
+	// TODO(random-liu): Introduce an intermediate state to avoid container creation after
+	// this point.
+	cntrs := c.containerStore.List()
+	for _, cntr := range cntrs {
+		if cntr.SandboxID != id {
+			continue
+		}
+		_, err = c.RemoveContainer(ctx, &runtime.RemoveContainerRequest{ContainerId: cntr.ID})
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to remove container %q", cntr.ID)
+		}
+	}
+
+	// Cleanup the sandbox root directories.
+	sandboxRootDir := c.getSandboxRootDir(id)
+	if err := ensureRemoveAll(ctx, sandboxRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove sandbox root directory %q",
+			sandboxRootDir)
+	}
+	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
+	if err := ensureRemoveAll(ctx, volatileSandboxRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove volatile sandbox root directory %q",
+			volatileSandboxRootDir)
+	}
+
+	// Delete sandbox container.
+	if err := sandbox.Container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
+		if !errdefs.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "failed to delete sandbox container %q", id)
+		}
+		log.G(ctx).Tracef("Remove called for sandbox container %q that does not exist", id)
+	}
+
+	// Remove sandbox from sandbox store. Note that once the sandbox is successfully
+	// deleted:
+	// 1) ListPodSandbox will not include this sandbox.
+	// 2) PodSandboxStatus and StopPodSandbox will return error.
+	// 3) On-going operations which have held the reference will not be affected.
+	c.sandboxStore.Delete(id)
+
+	// Release the sandbox name reserved for the sandbox.
+	c.sandboxNameIndex.ReleaseByKey(id)
+
+	return &runtime.RemovePodSandboxResponse{}, nil
+}

pkg/cri/server/sandbox_run.go

@@ -0,0 +1,548 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+	"math"
+	goruntime "runtime"
+	"strings"
+
+	"github.com/containerd/containerd"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	cni "github.com/containerd/go-cni"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+	"github.com/containerd/typeurl"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	"github.com/containerd/containerd/pkg/cri/server/bandwidth"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	"github.com/containerd/containerd/pkg/netns"
+	"github.com/containerd/containerd/pkg/util"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+)
+
+func init() {
+	typeurl.Register(&sandboxstore.Metadata{},
+		"github.com/containerd/cri/pkg/store/sandbox", "Metadata")
+}
+
+// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
+// the sandbox is in ready state.
+func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandboxRequest) (_ *runtime.RunPodSandboxResponse, retErr error) {
+	config := r.GetConfig()
+	log.G(ctx).Debugf("Sandbox config %+v", config)
+
+	// Generate unique id and name for the sandbox and reserve the name.
+	id := util.GenerateID()
+	metadata := config.GetMetadata()
+	if metadata == nil {
+		return nil, errors.New("sandbox config must include metadata")
+	}
+	name := makeSandboxName(metadata)
+	log.G(ctx).Debugf("Generated id %q for sandbox %q", id, name)
+	// Reserve the sandbox name to avoid concurrent `RunPodSandbox` request starting the
+	// same sandbox.
+	if err := c.sandboxNameIndex.Reserve(name, id); err != nil {
+		return nil, errors.Wrapf(err, "failed to reserve sandbox name %q", name)
+	}
+	defer func() {
+		// Release the name if the function returns with an error.
+		if retErr != nil {
+			c.sandboxNameIndex.ReleaseByName(name)
+		}
+	}()
+
+	// Create initial internal sandbox object.
+	sandbox := sandboxstore.NewSandbox(
+		sandboxstore.Metadata{
+			ID:             id,
+			Name:           name,
+			Config:         config,
+			RuntimeHandler: r.GetRuntimeHandler(),
+		},
+		sandboxstore.Status{
+			State: sandboxstore.StateUnknown,
+		},
+	)
+
+	// Ensure sandbox container image snapshot.
+	image, err := c.ensureImageExists(ctx, c.config.SandboxImage, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get sandbox image %q", c.config.SandboxImage)
+	}
+	containerdImage, err := c.toContainerdImage(ctx, *image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get image from containerd %q", image.ID)
+	}
+
+	ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox runtime")
+	}
+	log.G(ctx).Debugf("Use OCI %+v for sandbox %q", ociRuntime, id)
+
+	podNetwork := true
+	// Pod network is always needed on windows.
+	if goruntime.GOOS != "windows" &&
+		config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtime.NamespaceMode_NODE {
+		// Pod network is not needed on linux with host network.
+		podNetwork = false
+	}
+	if podNetwork {
+		// If it is not in host network namespace then create a namespace and set the sandbox
+		// handle. NetNSPath in sandbox metadata and NetNS is non empty only for non host network
+		// namespaces. If the pod is in host network namespace then both are empty and should not
+		// be used.
+		sandbox.NetNS, err = netns.NewNetNS()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create network namespace for sandbox %q", id)
+		}
+		sandbox.NetNSPath = sandbox.NetNS.GetPath()
+		defer func() {
+			if retErr != nil {
+				// Teardown network if an error is returned.
+				if err := c.teardownPodNetwork(ctx, sandbox); err != nil {
+					log.G(ctx).WithError(err).Errorf("Failed to destroy network for sandbox %q", id)
+				}
+
+				if err := sandbox.NetNS.Remove(); err != nil {
+					log.G(ctx).WithError(err).Errorf("Failed to remove network namespace %s for sandbox %q", sandbox.NetNSPath, id)
+				}
+				sandbox.NetNSPath = ""
+			}
+		}()
+
+		// Setup network for sandbox.
+		// Certain VM based solutions like clear containers (Issue containerd/cri-containerd#524)
+		// rely on the assumption that CRI shim will not be querying the network namespace to check the
+		// network states such as IP.
+		// In future runtime implementation should avoid relying on CRI shim implementation details.
+		// In this case however caching the IP will add a subtle performance enhancement by avoiding
+		// calls to network namespace of the pod to query the IP of the veth interface on every
+		// SandboxStatus request.
+		if err := c.setupPodNetwork(ctx, &sandbox); err != nil {
+			return nil, errors.Wrapf(err, "failed to setup network for sandbox %q", id)
+		}
+	}
+
+	// Create sandbox container.
+	// NOTE: sandboxContainerSpec SHOULD NOT have side
+	// effect, e.g. accessing/creating files, so that we can test
+	// it safely.
+	spec, err := c.sandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath, ociRuntime.PodAnnotations)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate sandbox container spec")
+	}
+	log.G(ctx).Debugf("Sandbox container %q spec: %#+v", id, spew.NewFormatter(spec))
+	sandbox.ProcessLabel = spec.Process.SelinuxLabel
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(sandbox.ProcessLabel)
+		}
+	}()
+
+	// handle any KVM based runtime
+	if err := modifyProcessLabel(ociRuntime.Type, spec); err != nil {
+		return nil, err
+	}
+
+	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+		// If privileged don't set selinux label, but we still record the MCS label so that
+		// the unused label can be freed later.
+		spec.Process.SelinuxLabel = ""
+	}
+
+	// Generate spec options that will be applied to the spec later.
+	specOpts, err := c.sandboxContainerSpecOpts(config, &image.ImageSpec.Config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate sanbdox container spec options")
+	}
+
+	sandboxLabels := buildLabels(config.Labels, containerKindSandbox)
+
+	runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate runtime options")
+	}
+	opts := []containerd.NewContainerOpts{
+		containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
+		customopts.WithNewSnapshot(id, containerdImage),
+		containerd.WithSpec(spec, specOpts...),
+		containerd.WithContainerLabels(sandboxLabels),
+		containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
+		containerd.WithRuntime(ociRuntime.Type, runtimeOpts)}
+
+	container, err := c.client.NewContainer(ctx, id, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd container")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			if err := container.Delete(deferCtx, containerd.WithSnapshotCleanup); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd container %q", id)
+			}
+		}
+	}()
+
+	// Create sandbox container root directories.
+	sandboxRootDir := c.getSandboxRootDir(id)
+	if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create sandbox root directory %q",
+			sandboxRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the sandbox root directory.
+			if err := c.os.RemoveAll(sandboxRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove sandbox root directory %q",
+					sandboxRootDir)
+			}
+		}
+	}()
+	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
+	if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create volatile sandbox root directory %q",
+			volatileSandboxRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile sandbox root directory.
+			if err := c.os.RemoveAll(volatileSandboxRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove volatile sandbox root directory %q",
+					volatileSandboxRootDir)
+			}
+		}
+	}()
+
+	// Setup files required for the sandbox.
+	if err = c.setupSandboxFiles(id, config); err != nil {
+		return nil, errors.Wrapf(err, "failed to setup sandbox files")
+	}
+	defer func() {
+		if retErr != nil {
+			if err = c.cleanupSandboxFiles(id, config); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to cleanup sandbox files in %q",
+					sandboxRootDir)
+			}
+		}
+	}()
+
+	// Update sandbox created timestamp.
+	info, err := container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox container info")
+	}
+
+	// Create sandbox task in containerd.
+	log.G(ctx).Tracef("Create sandbox container (id=%q, name=%q).",
+		id, name)
+
+	taskOpts := c.taskOpts(ociRuntime.Type)
+	// We don't need stdio for sandbox container.
+	task, err := container.NewTask(ctx, containerdio.NullIO, taskOpts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create containerd task")
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// Cleanup the sandbox container if an error is returned.
+			if _, err := task.Delete(deferCtx, WithNRISandboxDelete(id), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+				log.G(ctx).WithError(err).Errorf("Failed to delete sandbox container %q", id)
+			}
+		}
+	}()
+
+	// wait is a long running background request, no timeout needed.
+	exitCh, err := task.Wait(ctrdutil.NamespacedContext())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to wait for sandbox container task")
+	}
+
+	nric, err := nri.New()
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to create nri client")
+	}
+	if nric != nil {
+		nriSB := &nri.Sandbox{
+			ID:     id,
+			Labels: config.Labels,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil {
+			return nil, errors.Wrap(err, "nri invoke")
+		}
+	}
+
+	if err := task.Start(ctx); err != nil {
+		return nil, errors.Wrapf(err, "failed to start sandbox container task %q", id)
+	}
+
+	if err := sandbox.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
+		// Set the pod sandbox as ready after successfully start sandbox container.
+		status.Pid = task.Pid()
+		status.State = sandboxstore.StateReady
+		status.CreatedAt = info.CreatedAt
+		return status, nil
+	}); err != nil {
+		return nil, errors.Wrap(err, "failed to update sandbox status")
+	}
+
+	// Add sandbox into sandbox store in INIT state.
+	sandbox.Container = container
+
+	if err := c.sandboxStore.Add(sandbox); err != nil {
+		return nil, errors.Wrapf(err, "failed to add sandbox %+v into store", sandbox)
+	}
+
+	// start the monitor after adding sandbox into the store, this ensures
+	// that sandbox is in the store, when event monitor receives the TaskExit event.
+	//
+	// TaskOOM from containerd may come before sandbox is added to store,
+	// but we don't care about sandbox TaskOOM right now, so it is fine.
+	c.eventMonitor.startExitMonitor(context.Background(), id, task.Pid(), exitCh)
+
+	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
+}
+
+// setupPodNetwork setups up the network for a pod
+func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.Sandbox) error {
+	var (
+		id     = sandbox.ID
+		config = sandbox.Config
+		path   = sandbox.NetNSPath
+	)
+	if c.netPlugin == nil {
+		return errors.New("cni config not initialized")
+	}
+
+	opts, err := cniNamespaceOpts(id, config)
+	if err != nil {
+		return errors.Wrap(err, "get cni namespace options")
+	}
+
+	result, err := c.netPlugin.Setup(ctx, id, path, opts...)
+	if err != nil {
+		return err
+	}
+	logDebugCNIResult(ctx, id, result)
+	// Check if the default interface has IP config
+	if configs, ok := result.Interfaces[defaultIfName]; ok && len(configs.IPConfigs) > 0 {
+		sandbox.IP, sandbox.AdditionalIPs = selectPodIPs(configs.IPConfigs)
+		sandbox.CNIResult = result
+		return nil
+	}
+	return errors.Errorf("failed to find network info for sandbox %q", id)
+}
+
+// cniNamespaceOpts get CNI namespace options from sandbox config.
+func cniNamespaceOpts(id string, config *runtime.PodSandboxConfig) ([]cni.NamespaceOpts, error) {
+	opts := []cni.NamespaceOpts{
+		cni.WithLabels(toCNILabels(id, config)),
+	}
+
+	portMappings := toCNIPortMappings(config.GetPortMappings())
+	if len(portMappings) > 0 {
+		opts = append(opts, cni.WithCapabilityPortMap(portMappings))
+	}
+
+	// Will return an error if the bandwidth limitation has the wrong unit
+	// or an unreasonable value see validateBandwidthIsReasonable()
+	bandWidth, err := toCNIBandWidth(config.Annotations)
+	if err != nil {
+		return nil, err
+	}
+	if bandWidth != nil {
+		opts = append(opts, cni.WithCapabilityBandWidth(*bandWidth))
+	}
+
+	dns := toCNIDNS(config.GetDnsConfig())
+	if dns != nil {
+		opts = append(opts, cni.WithCapabilityDNS(*dns))
+	}
+
+	return opts, nil
+}
+
+// toCNILabels adds pod metadata into CNI labels.
+func toCNILabels(id string, config *runtime.PodSandboxConfig) map[string]string {
+	return map[string]string{
+		"K8S_POD_NAMESPACE":          config.GetMetadata().GetNamespace(),
+		"K8S_POD_NAME":               config.GetMetadata().GetName(),
+		"K8S_POD_INFRA_CONTAINER_ID": id,
+		"IgnoreUnknown":              "1",
+	}
+}
+
+// toCNIBandWidth converts CRI annotations to CNI bandwidth.
+func toCNIBandWidth(annotations map[string]string) (*cni.BandWidth, error) {
+	ingress, egress, err := bandwidth.ExtractPodBandwidthResources(annotations)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading pod bandwidth annotations")
+	}
+
+	if ingress == nil && egress == nil {
+		return nil, nil
+	}
+
+	bandWidth := &cni.BandWidth{}
+
+	if ingress != nil {
+		bandWidth.IngressRate = uint64(ingress.Value())
+		bandWidth.IngressBurst = math.MaxUint32
+	}
+
+	if egress != nil {
+		bandWidth.EgressRate = uint64(egress.Value())
+		bandWidth.EgressBurst = math.MaxUint32
+	}
+
+	return bandWidth, nil
+}
+
+// toCNIPortMappings converts CRI port mappings to CNI.
+func toCNIPortMappings(criPortMappings []*runtime.PortMapping) []cni.PortMapping {
+	var portMappings []cni.PortMapping
+	for _, mapping := range criPortMappings {
+		if mapping.HostPort <= 0 {
+			continue
+		}
+		portMappings = append(portMappings, cni.PortMapping{
+			HostPort:      mapping.HostPort,
+			ContainerPort: mapping.ContainerPort,
+			Protocol:      strings.ToLower(mapping.Protocol.String()),
+			HostIP:        mapping.HostIp,
+		})
+	}
+	return portMappings
+}
+
+// toCNIDNS converts CRI DNSConfig to CNI.
+func toCNIDNS(dns *runtime.DNSConfig) *cni.DNS {
+	if dns == nil {
+		return nil
+	}
+	return &cni.DNS{
+		Servers:  dns.GetServers(),
+		Searches: dns.GetSearches(),
+		Options:  dns.GetOptions(),
+	}
+}
+
+// selectPodIPs select an ip from the ip list. It prefers ipv4 more than ipv6
+// and returns the additional ips
+// TODO(random-liu): Revisit the ip order in the ipv6 beta stage. (cri#1278)
+func selectPodIPs(ipConfigs []*cni.IPConfig) (string, []string) {
+	var (
+		additionalIPs []string
+		ip            string
+	)
+	for _, c := range ipConfigs {
+		if c.IP.To4() != nil && ip == "" {
+			ip = c.IP.String()
+		} else {
+			additionalIPs = append(additionalIPs, c.IP.String())
+		}
+	}
+	if ip != "" {
+		return ip, additionalIPs
+	}
+	if len(ipConfigs) == 1 {
+		return additionalIPs[0], nil
+	}
+	return additionalIPs[0], additionalIPs[1:]
+}
+
+// untrustedWorkload returns true if the sandbox contains untrusted workload.
+func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
+	return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
+}
+
+// hostAccessingSandbox returns true if the sandbox configuration
+// requires additional host access for the sandbox.
+func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
+	securityContext := config.GetLinux().GetSecurityContext()
+
+	namespaceOptions := securityContext.GetNamespaceOptions()
+	if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
+		namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
+		namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		return true
+	}
+
+	return false
+}
+
+// getSandboxRuntime returns the runtime configuration for sandbox.
+// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned,
+// or else default runtime will be returned.
+func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
+	if untrustedWorkload(config) {
+		// If the untrusted annotation is provided, runtimeHandler MUST be empty.
+		if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
+			return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
+		}
+
+		//  If the untrusted workload is requesting access to the host/node, this request will fail.
+		//
+		//  Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
+		// runtime may support this.  For example, in a virtual-machine isolated runtime, privileged
+		// is a supported option, granting the workload to access the entire guest VM instead of host.
+		// TODO(windows): Deprecate this so that we don't need to handle it for windows.
+		if hostAccessingSandbox(config) {
+			return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
+		}
+
+		runtimeHandler = criconfig.RuntimeUntrusted
+	}
+
+	if runtimeHandler == "" {
+		runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
+	}
+
+	handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
+	if !ok {
+		return criconfig.Runtime{}, errors.Errorf("no runtime for %q is configured", runtimeHandler)
+	}
+	return handler, nil
+}
+
+func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.CNIResult) {
+	if logrus.GetLevel() < logrus.DebugLevel {
+		return
+	}
+	cniResult, err := json.Marshal(result)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("Failed to marshal CNI result for sandbox %q: %v", sandboxID, err)
+		return
+	}
+	log.G(ctx).Debugf("cni result for sandbox %q: %s", sandboxID, string(cniResult))
+}

pkg/cri/server/sandbox_run_linux.go

@@ -0,0 +1,308 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/plugin"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+	osinterface "github.com/containerd/containerd/pkg/os"
+)
+
+func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
+	// Creates a spec Generator with the default spec.
+	// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
+	specOpts := []oci.SpecOpts{
+		customopts.WithoutRunMount,
+		customopts.WithoutDefaultSecuritySettings,
+		customopts.WithRelativeRoot(relativeRootfsPath),
+		oci.WithEnv(imageConfig.Env),
+		oci.WithRootFSReadonly(),
+		oci.WithHostname(config.GetHostname()),
+	}
+	if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
+		// Pause image must have entrypoint or cmd.
+		return nil, errors.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
+	}
+	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
+
+	// Set cgroups parent.
+	if c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDisabledCgroups)
+	} else {
+		if config.GetLinux().GetCgroupParent() != "" {
+			cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id)
+			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
+		}
+	}
+
+	// When cgroup parent is not set, containerd-shim will create container in a child cgroup
+	// of the cgroup itself is in.
+	// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
+
+	// Set namespace options.
+	var (
+		securityContext = config.GetLinux().GetSecurityContext()
+		nsOptions       = securityContext.GetNamespaceOptions()
+	)
+	if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace))
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.UTSNamespace))
+	} else {
+		specOpts = append(specOpts, oci.WithLinuxNamespace(
+			runtimespec.LinuxNamespace{
+				Type: runtimespec.NetworkNamespace,
+				Path: nsPath,
+			}))
+	}
+	if nsOptions.GetPid() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace))
+	}
+	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace))
+	}
+
+	// It's fine to generate the spec before the sandbox /dev/shm
+	// is actually created.
+	sandboxDevShm := c.getSandboxDevShm(id)
+	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		sandboxDevShm = devShm
+	}
+	specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{
+		{
+			Source:      sandboxDevShm,
+			Destination: devShm,
+			Type:        "bind",
+			Options:     []string{"rbind", "ro"},
+		},
+		// Add resolv.conf for katacontainers to setup the DNS of pod VM properly.
+		{
+			Source:      c.getResolvPath(id),
+			Destination: resolvConfPath,
+			Type:        "bind",
+			Options:     []string{"rbind", "ro"},
+		},
+	}))
+
+	processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to init selinux options %+v", securityContext.GetSelinuxOptions())
+	}
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(processLabel)
+		}
+	}()
+
+	supplementalGroups := securityContext.GetSupplementalGroups()
+	specOpts = append(specOpts,
+		customopts.WithSelinuxLabels(processLabel, mountLabel),
+		customopts.WithSupplementalGroups(supplementalGroups),
+	)
+
+	// Add sysctls
+	sysctls := config.GetLinux().GetSysctls()
+	specOpts = append(specOpts, customopts.WithSysctls(sysctls))
+
+	// Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile
+
+	if !c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
+	}
+	specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		runtimePodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
+		customopts.WithAnnotation(annotations.SandboxID, id),
+		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
+	)
+
+	return c.runtimeSpec(id, "", specOpts...)
+}
+
+// sandboxContainerSpecOpts generates OCI spec options for
+// the sandbox container.
+func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	var (
+		securityContext = config.GetLinux().GetSecurityContext()
+		specOpts        []oci.SpecOpts
+	)
+	seccompSpecOpts, err := c.generateSeccompSpecOpts(
+		securityContext.GetSeccompProfilePath(),
+		securityContext.GetPrivileged(),
+		c.seccompEnabled())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate seccomp spec opts")
+	}
+	if seccompSpecOpts != nil {
+		specOpts = append(specOpts, seccompSpecOpts)
+	}
+
+	userstr, err := generateUserString(
+		"",
+		securityContext.GetRunAsUser(),
+		securityContext.GetRunAsGroup(),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate user string")
+	}
+	if userstr == "" {
+		// Lastly, since no user override was passed via CRI try to set via OCI
+		// Image
+		userstr = imageConfig.User
+	}
+	if userstr != "" {
+		specOpts = append(specOpts, oci.WithUser(userstr))
+	}
+	return specOpts, nil
+}
+
+// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
+// /etc/resolv.conf and /etc/hostname.
+func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	sandboxEtcHostname := c.getSandboxHostname(id)
+	hostname := config.GetHostname()
+	if hostname == "" {
+		var err error
+		hostname, err = c.os.Hostname()
+		if err != nil {
+			return errors.Wrap(err, "failed to get hostname")
+		}
+	}
+	if err := c.os.WriteFile(sandboxEtcHostname, []byte(hostname+"\n"), 0644); err != nil {
+		return errors.Wrapf(err, "failed to write hostname to %q", sandboxEtcHostname)
+	}
+
+	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
+	sandboxEtcHosts := c.getSandboxHosts(id)
+	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
+		return errors.Wrapf(err, "failed to generate sandbox hosts file %q", sandboxEtcHosts)
+	}
+
+	// Set DNS options. Maintain a resolv.conf for the sandbox.
+	var err error
+	resolvContent := ""
+	if dnsConfig := config.GetDnsConfig(); dnsConfig != nil {
+		resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse sandbox DNSConfig %+v", dnsConfig)
+		}
+	}
+	resolvPath := c.getResolvPath(id)
+	if resolvContent == "" {
+		// copy host's resolv.conf to resolvPath
+		err = c.os.CopyFile(resolvConfPath, resolvPath, 0644)
+		if err != nil {
+			return errors.Wrapf(err, "failed to copy host's resolv.conf to %q", resolvPath)
+		}
+	} else {
+		err = c.os.WriteFile(resolvPath, []byte(resolvContent), 0644)
+		if err != nil {
+			return errors.Wrapf(err, "failed to write resolv content to %q", resolvPath)
+		}
+	}
+
+	// Setup sandbox /dev/shm.
+	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
+		if _, err := c.os.Stat(devShm); err != nil {
+			return errors.Wrapf(err, "host %q is not available for host ipc", devShm)
+		}
+	} else {
+		sandboxDevShm := c.getSandboxDevShm(id)
+		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
+			return errors.Wrap(err, "failed to create sandbox shm")
+		}
+		shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
+		if err := c.os.(osinterface.UNIX).Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
+			return errors.Wrap(err, "failed to mount sandbox shm")
+		}
+	}
+
+	return nil
+}
+
+// parseDNSOptions parse DNS options into resolv.conf format content,
+// if none option is specified, will return empty with no error.
+func parseDNSOptions(servers, searches, options []string) (string, error) {
+	resolvContent := ""
+
+	if len(searches) > maxDNSSearches {
+		return "", errors.Errorf("DNSOption.Searches has more than %d domains", maxDNSSearches)
+	}
+
+	if len(searches) > 0 {
+		resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " "))
+	}
+
+	if len(servers) > 0 {
+		resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver "))
+	}
+
+	if len(options) > 0 {
+		resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " "))
+	}
+
+	return resolvContent, nil
+}
+
+// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
+// remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
+func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() != runtime.NamespaceMode_NODE {
+		path, err := c.os.FollowSymlinkInScope(c.getSandboxDevShm(id), "/")
+		if err != nil {
+			return errors.Wrap(err, "failed to follow symlink")
+		}
+		if err := c.os.(osinterface.UNIX).Unmount(path); err != nil && !os.IsNotExist(err) {
+			return errors.Wrapf(err, "failed to unmount %q", path)
+		}
+	}
+	return nil
+}
+
+// taskOpts generates task options for a (sandbox) container.
+func (c *criService) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	// TODO(random-liu): Remove this after shim v1 is deprecated.
+	var taskOpts []containerd.NewTaskOpts
+	if c.config.NoPivot && (runtimeType == plugin.RuntimeRuncV1 || runtimeType == plugin.RuntimeRuncV2) {
+		taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
+	}
+	return taskOpts
+}

pkg/cri/server/sandbox_run_linux_test.go

@@ -0,0 +1,431 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	"github.com/containerd/containerd/pkg/containerd/opts"
+	ostesting "github.com/containerd/containerd/pkg/os/testing"
+)
+
+func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
+	config := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-name",
+			Uid:       "test-uid",
+			Namespace: "test-ns",
+			Attempt:   1,
+		},
+		Hostname:     "test-hostname",
+		LogDirectory: "test-log-directory",
+		Labels:       map[string]string{"a": "b"},
+		Annotations:  map[string]string{"c": "d"},
+		Linux: &runtime.LinuxPodSandboxConfig{
+			CgroupParent: "/test/cgroup/parent",
+		},
+	}
+	imageConfig := &imagespec.ImageConfig{
+		Env:        []string{"a=b", "c=d"},
+		Entrypoint: []string{"/pause"},
+		Cmd:        []string{"forever"},
+		WorkingDir: "/workspace",
+	}
+	specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
+		assert.Equal(t, "test-hostname", spec.Hostname)
+		assert.Equal(t, getCgroupsPath("/test/cgroup/parent", id), spec.Linux.CgroupsPath)
+		assert.Equal(t, relativeRootfsPath, spec.Root.Path)
+		assert.Equal(t, true, spec.Root.Readonly)
+		assert.Contains(t, spec.Process.Env, "a=b", "c=d")
+		assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args)
+		assert.Equal(t, "/workspace", spec.Process.Cwd)
+		assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, opts.DefaultSandboxCPUshares)
+		assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj)
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox)
+
+		assert.Contains(t, spec.Annotations, annotations.SandboxLogDir)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory")
+
+		if selinux.GetEnabled() {
+			assert.NotEqual(t, "", spec.Process.SelinuxLabel)
+			assert.NotEqual(t, "", spec.Linux.MountLabel)
+		}
+	}
+	return config, imageConfig, specCheck
+}
+
+func TestLinuxSandboxContainerSpec(t *testing.T) {
+	testID := "test-id"
+	nsPath := "test-cni"
+	for desc, test := range map[string]struct {
+		configChange func(*runtime.PodSandboxConfig)
+		specCheck    func(*testing.T, *runtimespec.Spec)
+		expectErr    bool
+	}{
+		"spec should reflect original config": {
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				// runtime spec should have expected namespaces enabled by default.
+				require.NotNil(t, spec.Linux)
+				assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.NetworkNamespace,
+					Path: nsPath,
+				})
+				assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.UTSNamespace,
+				})
+				assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.PIDNamespace,
+				})
+				assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.IPCNamespace,
+				})
+			},
+		},
+		"host namespace": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						Network: runtime.NamespaceMode_NODE,
+						Pid:     runtime.NamespaceMode_NODE,
+						Ipc:     runtime.NamespaceMode_NODE,
+					},
+				}
+			},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				// runtime spec should disable expected namespaces in host mode.
+				require.NotNil(t, spec.Linux)
+				assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.NetworkNamespace,
+				})
+				assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.UTSNamespace,
+				})
+				assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.PIDNamespace,
+				})
+				assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
+					Type: runtimespec.IPCNamespace,
+				})
+			},
+		},
+		"should set supplemental groups correctly": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
+					SupplementalGroups: []int64{1111, 2222},
+				}
+			},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				require.NotNil(t, spec.Process)
+				assert.Contains(t, spec.Process.User.AdditionalGids, uint32(1111))
+				assert.Contains(t, spec.Process.User.AdditionalGids, uint32(2222))
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		config, imageConfig, specCheck := getRunPodSandboxTestData()
+		if test.configChange != nil {
+			test.configChange(config)
+		}
+		spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Nil(t, spec)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.NotNil(t, spec)
+		specCheck(t, testID, spec)
+		if test.specCheck != nil {
+			test.specCheck(t, spec)
+		}
+	}
+}
+
+func TestSetupSandboxFiles(t *testing.T) {
+	const (
+		testID       = "test-id"
+		realhostname = "test-real-hostname"
+	)
+	for desc, test := range map[string]struct {
+		dnsConfig     *runtime.DNSConfig
+		hostname      string
+		ipcMode       runtime.NamespaceMode
+		expectedCalls []ostesting.CalledDetail
+	}{
+		"should check host /dev/shm existence when ipc mode is NODE": {
+			ipcMode: runtime.NamespaceMode_NODE,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "Hostname",
+				},
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
+						[]byte(realhostname + "\n"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/resolv.conf",
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+		"should create new /etc/resolv.conf if DNSOptions is set": {
+			dnsConfig: &runtime.DNSConfig{
+				Servers:  []string{"8.8.8.8"},
+				Searches: []string{"114.114.114.114"},
+				Options:  []string{"timeout:1"},
+			},
+			ipcMode: runtime.NamespaceMode_NODE,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "Hostname",
+				},
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
+						[]byte(realhostname + "\n"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						[]byte(`search 114.114.114.114
+nameserver 8.8.8.8
+options timeout:1
+`), os.FileMode(0644),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+		"should create sandbox shm when ipc namespace mode is not NODE": {
+			ipcMode: runtime.NamespaceMode_POD,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "Hostname",
+				},
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
+						[]byte(realhostname + "\n"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/resolv.conf",
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "MkdirAll",
+					Arguments: []interface{}{
+						filepath.Join(testStateDir, sandboxesDir, testID, "shm"),
+						os.FileMode(0700),
+					},
+				},
+				{
+					Name: "Mount",
+					// Ignore arguments which are too complex to check.
+				},
+			},
+		},
+		"should create /etc/hostname when hostname is set": {
+			hostname: "test-hostname",
+			ipcMode:  runtime.NamespaceMode_NODE,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
+						[]byte("test-hostname\n"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/resolv.conf",
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						os.FileMode(0644),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		c.os.(*ostesting.FakeOS).HostnameFn = func() (string, error) {
+			return realhostname, nil
+		}
+		cfg := &runtime.PodSandboxConfig{
+			Hostname:  test.hostname,
+			DnsConfig: test.dnsConfig,
+			Linux: &runtime.LinuxPodSandboxConfig{
+				SecurityContext: &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						Ipc: test.ipcMode,
+					},
+				},
+			},
+		}
+		c.setupSandboxFiles(testID, cfg)
+		calls := c.os.(*ostesting.FakeOS).GetCalls()
+		assert.Len(t, calls, len(test.expectedCalls))
+		for i, expected := range test.expectedCalls {
+			if expected.Arguments == nil {
+				// Ignore arguments.
+				expected.Arguments = calls[i].Arguments
+			}
+			assert.Equal(t, expected, calls[i])
+		}
+	}
+}
+
+func TestParseDNSOption(t *testing.T) {
+	for desc, test := range map[string]struct {
+		servers         []string
+		searches        []string
+		options         []string
+		expectedContent string
+		expectErr       bool
+	}{
+		"empty dns options should return empty content": {},
+		"non-empty dns options should return correct content": {
+			servers:  []string{"8.8.8.8", "server.google.com"},
+			searches: []string{"114.114.114.114"},
+			options:  []string{"timeout:1"},
+			expectedContent: `search 114.114.114.114
+nameserver 8.8.8.8
+nameserver server.google.com
+options timeout:1
+`,
+		},
+		"should return error if dns search exceeds limit(6)": {
+			searches: []string{
+				"server0.google.com",
+				"server1.google.com",
+				"server2.google.com",
+				"server3.google.com",
+				"server4.google.com",
+				"server5.google.com",
+				"server6.google.com",
+			},
+			expectErr: true,
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		resolvContent, err := parseDNSOptions(test.servers, test.searches, test.options)
+		if test.expectErr {
+			assert.Error(t, err)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.Equal(t, resolvContent, test.expectedContent)
+	}
+}
+
+func TestSandboxDisableCgroup(t *testing.T) {
+	config, imageConfig, _ := getRunPodSandboxTestData()
+	c := newTestCRIService()
+	c.config.DisableCgroup = true
+	spec, err := c.sandboxContainerSpec("test-id", config, imageConfig, "test-cni", []string{})
+	require.NoError(t, err)
+
+	t.Log("resource limit should not be set")
+	assert.Nil(t, spec.Linux.Resources.Memory)
+	assert.Nil(t, spec.Linux.Resources.CPU)
+
+	t.Log("cgroup path should be empty")
+	assert.Empty(t, spec.Linux.CgroupsPath)
+}
+
+// TODO(random-liu): [P1] Add unit test for different error cases to make sure
+// the function cleans up on error properly.

pkg/cri/server/sandbox_run_other.go

@@ -0,0 +1,55 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
+	return c.runtimeSpec(id, "")
+}
+
+// sandboxContainerSpecOpts generates OCI spec options for
+// the sandbox container.
+func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return []oci.SpecOpts{}, nil
+}
+
+// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
+// /etc/resolv.conf and /etc/hostname.
+func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
+// remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
+func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// taskOpts generates task options for a (sandbox) container.
+func (c *criService) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	return []containerd.NewTaskOpts{}
+}

pkg/cri/server/sandbox_run_other_test.go

@@ -0,0 +1,35 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
+	config := &runtime.PodSandboxConfig{}
+	imageConfig := &imagespec.ImageConfig{}
+	specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
+	}
+	return config, imageConfig, specCheck
+}

pkg/cri/server/sandbox_run_test.go

@@ -0,0 +1,500 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"net"
+	"testing"
+
+	cni "github.com/containerd/go-cni"
+	"github.com/containerd/typeurl"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestSandboxContainerSpec(t *testing.T) {
+	testID := "test-id"
+	nsPath := "test-cni"
+	for desc, test := range map[string]struct {
+		configChange      func(*runtime.PodSandboxConfig)
+		podAnnotations    []string
+		imageConfigChange func(*imagespec.ImageConfig)
+		specCheck         func(*testing.T, *runtimespec.Spec)
+		expectErr         bool
+	}{
+		"should return error when entrypoint and cmd are empty": {
+			imageConfigChange: func(c *imagespec.ImageConfig) {
+				c.Entrypoint = nil
+				c.Cmd = nil
+			},
+			expectErr: true,
+		},
+		"a passthrough annotation should be passed as an OCI annotation": {
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+			},
+		},
+		"a non-passthrough annotation should not be passed as an OCI annotation": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["d"] = "e"
+			},
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+				_, ok := spec.Annotations["d"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations should support wildcard match": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["t.f"] = "j"
+				c.Annotations["z.g"] = "o"
+				c.Annotations["z"] = "o"
+				c.Annotations["y.ca"] = "b"
+				c.Annotations["y"] = "b"
+			},
+			podAnnotations: []string{"t*", "z.*", "y.c*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["t.f"], "j")
+				assert.Equal(t, spec.Annotations["z.g"], "o")
+				assert.Equal(t, spec.Annotations["y.ca"], "b")
+				_, ok := spec.Annotations["y"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["z"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		c := newTestCRIService()
+		config, imageConfig, specCheck := getRunPodSandboxTestData()
+		if test.configChange != nil {
+			test.configChange(config)
+		}
+
+		if test.imageConfigChange != nil {
+			test.imageConfigChange(imageConfig)
+		}
+		spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath,
+			test.podAnnotations)
+		if test.expectErr {
+			assert.Error(t, err)
+			assert.Nil(t, spec)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.NotNil(t, spec)
+		specCheck(t, testID, spec)
+		if test.specCheck != nil {
+			test.specCheck(t, spec)
+		}
+	}
+}
+
+func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
+	for desc, test := range map[string]struct {
+		configChange func(*runtime.PodSandboxConfig)
+	}{
+		"should marshal original config": {},
+		"should marshal Linux": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				if c.Linux == nil {
+					c.Linux = &runtime.LinuxPodSandboxConfig{}
+				}
+				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						Network: runtime.NamespaceMode_NODE,
+						Pid:     runtime.NamespaceMode_NODE,
+						Ipc:     runtime.NamespaceMode_NODE,
+					},
+					SupplementalGroups: []int64{1111, 2222},
+				}
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		meta := &sandboxstore.Metadata{
+			ID:        "1",
+			Name:      "sandbox_1",
+			NetNSPath: "/home/cloud",
+		}
+		meta.Config, _, _ = getRunPodSandboxTestData()
+		if test.configChange != nil {
+			test.configChange(meta.Config)
+		}
+
+		any, err := typeurl.MarshalAny(meta)
+		assert.NoError(t, err)
+		data, err := typeurl.UnmarshalAny(any)
+		assert.NoError(t, err)
+		assert.IsType(t, &sandboxstore.Metadata{}, data)
+		curMeta, ok := data.(*sandboxstore.Metadata)
+		assert.True(t, ok)
+		assert.Equal(t, meta, curMeta)
+	}
+}
+
+func TestToCNIPortMappings(t *testing.T) {
+	for desc, test := range map[string]struct {
+		criPortMappings []*runtime.PortMapping
+		cniPortMappings []cni.PortMapping
+	}{
+		"empty CRI port mapping should map to empty CNI port mapping": {},
+		"CRI port mapping should be converted to CNI port mapping properly": {
+			criPortMappings: []*runtime.PortMapping{
+				{
+					Protocol:      runtime.Protocol_UDP,
+					ContainerPort: 1234,
+					HostPort:      5678,
+					HostIp:        "123.124.125.126",
+				},
+				{
+					Protocol:      runtime.Protocol_TCP,
+					ContainerPort: 4321,
+					HostPort:      8765,
+					HostIp:        "126.125.124.123",
+				},
+				{
+					Protocol:      runtime.Protocol_SCTP,
+					ContainerPort: 1234,
+					HostPort:      5678,
+					HostIp:        "123.124.125.126",
+				},
+			},
+			cniPortMappings: []cni.PortMapping{
+				{
+					HostPort:      5678,
+					ContainerPort: 1234,
+					Protocol:      "udp",
+					HostIP:        "123.124.125.126",
+				},
+				{
+					HostPort:      8765,
+					ContainerPort: 4321,
+					Protocol:      "tcp",
+					HostIP:        "126.125.124.123",
+				},
+				{
+					HostPort:      5678,
+					ContainerPort: 1234,
+					Protocol:      "sctp",
+					HostIP:        "123.124.125.126",
+				},
+			},
+		},
+		"CRI port mapping without host port should be skipped": {
+			criPortMappings: []*runtime.PortMapping{
+				{
+					Protocol:      runtime.Protocol_UDP,
+					ContainerPort: 1234,
+					HostIp:        "123.124.125.126",
+				},
+				{
+					Protocol:      runtime.Protocol_TCP,
+					ContainerPort: 4321,
+					HostPort:      8765,
+					HostIp:        "126.125.124.123",
+				},
+			},
+			cniPortMappings: []cni.PortMapping{
+				{
+					HostPort:      8765,
+					ContainerPort: 4321,
+					Protocol:      "tcp",
+					HostIP:        "126.125.124.123",
+				},
+			},
+		},
+		"CRI port mapping with unsupported protocol should be skipped": {
+			criPortMappings: []*runtime.PortMapping{
+				{
+					Protocol:      runtime.Protocol_TCP,
+					ContainerPort: 4321,
+					HostPort:      8765,
+					HostIp:        "126.125.124.123",
+				},
+			},
+			cniPortMappings: []cni.PortMapping{
+				{
+					HostPort:      8765,
+					ContainerPort: 4321,
+					Protocol:      "tcp",
+					HostIP:        "126.125.124.123",
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		assert.Equal(t, test.cniPortMappings, toCNIPortMappings(test.criPortMappings))
+	}
+}
+
+func TestSelectPodIP(t *testing.T) {
+	for desc, test := range map[string]struct {
+		ips                   []string
+		expectedIP            string
+		expectedAdditionalIPs []string
+	}{
+		"ipv4 should be picked even if ipv6 comes first": {
+			ips:                   []string{"2001:db8:85a3::8a2e:370:7334", "192.168.17.43"},
+			expectedIP:            "192.168.17.43",
+			expectedAdditionalIPs: []string{"2001:db8:85a3::8a2e:370:7334"},
+		},
+		"ipv4 should be picked when there is only ipv4": {
+			ips:                   []string{"192.168.17.43"},
+			expectedIP:            "192.168.17.43",
+			expectedAdditionalIPs: nil,
+		},
+		"ipv6 should be picked when there is no ipv4": {
+			ips:                   []string{"2001:db8:85a3::8a2e:370:7334"},
+			expectedIP:            "2001:db8:85a3::8a2e:370:7334",
+			expectedAdditionalIPs: nil,
+		},
+		"the first ipv4 should be picked when there are multiple ipv4": { // unlikely to happen
+			ips:                   []string{"2001:db8:85a3::8a2e:370:7334", "192.168.17.43", "2001:db8:85a3::8a2e:370:7335", "192.168.17.45"},
+			expectedIP:            "192.168.17.43",
+			expectedAdditionalIPs: []string{"2001:db8:85a3::8a2e:370:7334", "2001:db8:85a3::8a2e:370:7335", "192.168.17.45"},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
+		var ipConfigs []*cni.IPConfig
+		for _, ip := range test.ips {
+			ipConfigs = append(ipConfigs, &cni.IPConfig{
+				IP: net.ParseIP(ip),
+			})
+		}
+		ip, additionalIPs := selectPodIPs(ipConfigs)
+		assert.Equal(t, test.expectedIP, ip)
+		assert.Equal(t, test.expectedAdditionalIPs, additionalIPs)
+	}
+}
+
+func TestHostAccessingSandbox(t *testing.T) {
+	privilegedContext := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: true,
+			},
+		},
+	}
+	nonPrivilegedContext := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: false,
+			},
+		},
+	}
+	hostNamespace := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: false,
+				NamespaceOptions: &runtime.NamespaceOption{
+					Network: runtime.NamespaceMode_NODE,
+					Pid:     runtime.NamespaceMode_NODE,
+					Ipc:     runtime.NamespaceMode_NODE,
+				},
+			},
+		},
+	}
+	tests := []struct {
+		name   string
+		config *runtime.PodSandboxConfig
+		want   bool
+	}{
+		{"Security Context is nil", nil, false},
+		{"Security Context is privileged", privilegedContext, false},
+		{"Security Context is not privileged", nonPrivilegedContext, false},
+		{"Security Context namespace host access", hostNamespace, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := hostAccessingSandbox(tt.config); got != tt.want {
+				t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetSandboxRuntime(t *testing.T) {
+	untrustedWorkloadRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "untrusted-workload-runtime",
+		Root:   "",
+	}
+
+	defaultRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "default-runtime",
+		Root:   "",
+	}
+
+	fooRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "foo-bar",
+		Root:   "",
+	}
+
+	for desc, test := range map[string]struct {
+		sandboxConfig   *runtime.PodSandboxConfig
+		runtimeHandler  string
+		runtimes        map[string]criconfig.Runtime
+		expectErr       bool
+		expectedRuntime criconfig.Runtime
+	}{
+		"should return error if untrusted workload requires host access": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Linux: &runtime.LinuxPodSandboxConfig{
+					SecurityContext: &runtime.LinuxSandboxSecurityContext{
+						Privileged: false,
+						NamespaceOptions: &runtime.NamespaceOption{
+							Network: runtime.NamespaceMode_NODE,
+							Pid:     runtime.NamespaceMode_NODE,
+							Ipc:     runtime.NamespaceMode_NODE,
+						},
+					},
+				},
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectErr: true,
+		},
+		"should use untrusted workload runtime for untrusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should use default runtime for regular workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
+			expectedRuntime: defaultRuntime,
+		},
+		"should use default runtime for trusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "false",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: defaultRuntime,
+		},
+		"should return error if untrusted workload runtime is required but not configured": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
+			expectErr: true,
+		},
+		"should use 'untrusted' runtime for untrusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should use 'untrusted' runtime for untrusted workload & handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler: "untrusted",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should return an error if untrusted annotation with conflicting handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler: "foo",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
+			expectErr: true,
+		},
+		"should use correct runtime for a runtime handler": {
+			sandboxConfig:  &runtime.PodSandboxConfig{},
+			runtimeHandler: "foo",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
+			expectedRuntime: fooRuntime,
+		},
+		"should return error if runtime handler is required but not configured": {
+			sandboxConfig:  &runtime.PodSandboxConfig{},
+			runtimeHandler: "bar",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+				"foo":                    fooRuntime,
+			},
+			expectErr: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			cri := newTestCRIService()
+			cri.config = criconfig.Config{
+				PluginConfig: criconfig.DefaultConfig(),
+			}
+			cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
+			cri.config.ContainerdConfig.Runtimes = test.runtimes
+			r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
+			assert.Equal(t, test.expectErr, err != nil)
+			assert.Equal(t, test.expectedRuntime, r)
+		})
+	}
+}

pkg/cri/server/sandbox_run_windows.go

@@ -0,0 +1,91 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	customopts "github.com/containerd/containerd/pkg/containerd/opts"
+)
+
+func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
+	// Creates a spec Generator with the default spec.
+	specOpts := []oci.SpecOpts{
+		oci.WithEnv(imageConfig.Env),
+		oci.WithHostname(config.GetHostname()),
+	}
+	if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
+		// Pause image must have entrypoint or cmd.
+		return nil, errors.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
+	}
+	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
+
+	specOpts = append(specOpts,
+		// Clear the root location since hcsshim expects it.
+		// NOTE: readonly rootfs doesn't work on windows.
+		customopts.WithoutRoot,
+		customopts.WithWindowsNetworkNamespace(nsPath),
+	)
+
+	specOpts = append(specOpts, customopts.WithWindowsDefaultSandboxShares)
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		runtimePodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
+		customopts.WithAnnotation(annotations.SandboxID, id),
+		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
+	)
+
+	return c.runtimeSpec(id, "", specOpts...)
+}
+
+// No sandbox container spec options for windows yet.
+func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return nil, nil
+}
+
+// No sandbox files needed for windows.
+func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// No sandbox files needed for windows.
+func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// No task options needed for windows.
+func (c *criService) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	return nil
+}

pkg/cri/server/sandbox_run_windows_test.go

@@ -0,0 +1,86 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/annotations"
+	"github.com/containerd/containerd/pkg/containerd/opts"
+)
+
+func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
+	config := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-name",
+			Uid:       "test-uid",
+			Namespace: "test-ns",
+			Attempt:   1,
+		},
+		Hostname:     "test-hostname",
+		LogDirectory: "test-log-directory",
+		Labels:       map[string]string{"a": "b"},
+		Annotations:  map[string]string{"c": "d"},
+	}
+	imageConfig := &imagespec.ImageConfig{
+		Env:        []string{"a=b", "c=d"},
+		Entrypoint: []string{"/pause"},
+		Cmd:        []string{"forever"},
+		WorkingDir: "/workspace",
+	}
+	specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
+		assert.Equal(t, "test-hostname", spec.Hostname)
+		assert.Nil(t, spec.Root)
+		assert.Contains(t, spec.Process.Env, "a=b", "c=d")
+		assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args)
+		assert.Equal(t, "/workspace", spec.Process.Cwd)
+		assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, opts.DefaultSandboxCPUshares)
+
+		t.Logf("Check PodSandbox annotations")
+		assert.Contains(t, spec.Annotations, annotations.SandboxID)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id)
+
+		assert.Contains(t, spec.Annotations, annotations.ContainerType)
+		assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox)
+
+		assert.Contains(t, spec.Annotations, annotations.SandboxLogDir)
+		assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory")
+	}
+	return config, imageConfig, specCheck
+}
+
+func TestSandboxWindowsNetworkNamespace(t *testing.T) {
+	testID := "test-id"
+	nsPath := "test-cni"
+	c := newTestCRIService()
+
+	config, imageConfig, specCheck := getRunPodSandboxTestData()
+	spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, spec)
+	specCheck(t, testID, spec)
+	assert.NotNil(t, spec.Windows)
+	assert.NotNil(t, spec.Windows.Network)
+	assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace)
+}

pkg/cri/server/sandbox_status.go

@@ -0,0 +1,217 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+	goruntime "runtime"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	cni "github.com/containerd/go-cni"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+// PodSandboxStatus returns the status of the PodSandbox.
+func (c *criService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandboxStatusRequest) (*runtime.PodSandboxStatusResponse, error) {
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		return nil, errors.Wrap(err, "an error occurred when try to find sandbox")
+	}
+
+	ip, additionalIPs, err := c.getIPs(sandbox)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox ip")
+	}
+	status := toCRISandboxStatus(sandbox.Metadata, sandbox.Status.Get(), ip, additionalIPs)
+	if status.GetCreatedAt() == 0 {
+		// CRI doesn't allow CreatedAt == 0.
+		info, err := sandbox.Container.Info(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get CreatedAt for sandbox container in %q state", status.State)
+		}
+		status.CreatedAt = info.CreatedAt.UnixNano()
+	}
+	if !r.GetVerbose() {
+		return &runtime.PodSandboxStatusResponse{Status: status}, nil
+	}
+
+	// Generate verbose information.
+	info, err := toCRISandboxInfo(ctx, sandbox)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get verbose sandbox container info")
+	}
+
+	return &runtime.PodSandboxStatusResponse{
+		Status: status,
+		Info:   info,
+	}, nil
+}
+
+func (c *criService) getIPs(sandbox sandboxstore.Sandbox) (string, []string, error) {
+	config := sandbox.Config
+
+	if goruntime.GOOS != "windows" &&
+		config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtime.NamespaceMode_NODE {
+		// For sandboxes using the node network we are not
+		// responsible for reporting the IP.
+		return "", nil, nil
+	}
+
+	if closed, err := sandbox.NetNS.Closed(); err != nil {
+		return "", nil, errors.Wrap(err, "check network namespace closed")
+	} else if closed {
+		return "", nil, nil
+	}
+
+	return sandbox.IP, sandbox.AdditionalIPs, nil
+}
+
+// toCRISandboxStatus converts sandbox metadata into CRI pod sandbox status.
+func toCRISandboxStatus(meta sandboxstore.Metadata, status sandboxstore.Status, ip string, additionalIPs []string) *runtime.PodSandboxStatus {
+	// Set sandbox state to NOTREADY by default.
+	state := runtime.PodSandboxState_SANDBOX_NOTREADY
+	if status.State == sandboxstore.StateReady {
+		state = runtime.PodSandboxState_SANDBOX_READY
+	}
+	nsOpts := meta.Config.GetLinux().GetSecurityContext().GetNamespaceOptions()
+	var ips []*runtime.PodIP
+	for _, additionalIP := range additionalIPs {
+		ips = append(ips, &runtime.PodIP{Ip: additionalIP})
+	}
+	return &runtime.PodSandboxStatus{
+		Id:        meta.ID,
+		Metadata:  meta.Config.GetMetadata(),
+		State:     state,
+		CreatedAt: status.CreatedAt.UnixNano(),
+		Network: &runtime.PodSandboxNetworkStatus{
+			Ip:            ip,
+			AdditionalIps: ips,
+		},
+		Linux: &runtime.LinuxPodSandboxStatus{
+			Namespaces: &runtime.Namespace{
+				Options: &runtime.NamespaceOption{
+					Network: nsOpts.GetNetwork(),
+					Pid:     nsOpts.GetPid(),
+					Ipc:     nsOpts.GetIpc(),
+				},
+			},
+		},
+		Labels:         meta.Config.GetLabels(),
+		Annotations:    meta.Config.GetAnnotations(),
+		RuntimeHandler: meta.RuntimeHandler,
+	}
+}
+
+// SandboxInfo is extra information for sandbox.
+// TODO (mikebrow): discuss predefining constants structures for some or all of these field names in CRI
+type SandboxInfo struct {
+	Pid         uint32 `json:"pid"`
+	Status      string `json:"processStatus"`
+	NetNSClosed bool   `json:"netNamespaceClosed"`
+	Image       string `json:"image"`
+	SnapshotKey string `json:"snapshotKey"`
+	Snapshotter string `json:"snapshotter"`
+	// Note: a new field `RuntimeHandler` has been added into the CRI PodSandboxStatus struct, and
+	// should be set. This `RuntimeHandler` field will be deprecated after containerd 1.3 (tracked
+	// in https://github.com/containerd/cri/issues/1064).
+	RuntimeHandler string                    `json:"runtimeHandler"` // see the Note above
+	RuntimeType    string                    `json:"runtimeType"`
+	RuntimeOptions interface{}               `json:"runtimeOptions"`
+	Config         *runtime.PodSandboxConfig `json:"config"`
+	RuntimeSpec    *runtimespec.Spec         `json:"runtimeSpec"`
+	CNIResult      *cni.CNIResult            `json:"cniResult"`
+}
+
+// toCRISandboxInfo converts internal container object information to CRI sandbox status response info map.
+func toCRISandboxInfo(ctx context.Context, sandbox sandboxstore.Sandbox) (map[string]string, error) {
+	container := sandbox.Container
+	task, err := container.Task(ctx, nil)
+	if err != nil && !errdefs.IsNotFound(err) {
+		return nil, errors.Wrap(err, "failed to get sandbox container task")
+	}
+
+	var processStatus containerd.ProcessStatus
+	if task != nil {
+		taskStatus, err := task.Status(ctx)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get task status")
+		}
+
+		processStatus = taskStatus.Status
+	}
+
+	si := &SandboxInfo{
+		Pid:            sandbox.Status.Get().Pid,
+		RuntimeHandler: sandbox.RuntimeHandler,
+		Status:         string(processStatus),
+		Config:         sandbox.Config,
+		CNIResult:      sandbox.CNIResult,
+	}
+
+	if si.Status == "" {
+		// If processStatus is empty, it means that the task is deleted. Apply "deleted"
+		// status which does not exist in containerd.
+		si.Status = "deleted"
+	}
+
+	if sandbox.NetNS != nil {
+		// Add network closed information if sandbox is not using host network.
+		closed, err := sandbox.NetNS.Closed()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to check network namespace closed")
+		}
+		si.NetNSClosed = closed
+	}
+
+	spec, err := container.Spec(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox container runtime spec")
+	}
+	si.RuntimeSpec = spec
+
+	ctrInfo, err := container.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get sandbox container info")
+	}
+	// Do not use config.SandboxImage because the configuration might
+	// be changed during restart. It may not reflect the actual image
+	// used by the sandbox container.
+	si.Image = ctrInfo.Image
+	si.SnapshotKey = ctrInfo.SnapshotKey
+	si.Snapshotter = ctrInfo.Snapshotter
+
+	runtimeOptions, err := getRuntimeOptions(ctrInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get runtime options")
+	}
+	si.RuntimeType = ctrInfo.Runtime.Name
+	si.RuntimeOptions = runtimeOptions
+
+	infoBytes, err := json.Marshal(si)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to marshal info %v", si)
+	}
+	return map[string]string{
+		"info": string(infoBytes),
+	}, nil
+}

pkg/cri/server/sandbox_status_test.go

@@ -0,0 +1,116 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestPodSandboxStatus(t *testing.T) {
+	const (
+		id = "test-id"
+		ip = "10.10.10.10"
+	)
+	additionalIPs := []string{"8.8.8.8", "2001:db8:85a3::8a2e:370:7334"}
+	createdAt := time.Now()
+	config := &runtime.PodSandboxConfig{
+		Metadata: &runtime.PodSandboxMetadata{
+			Name:      "test-name",
+			Uid:       "test-uid",
+			Namespace: "test-ns",
+			Attempt:   1,
+		},
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				NamespaceOptions: &runtime.NamespaceOption{
+					Network: runtime.NamespaceMode_NODE,
+					Pid:     runtime.NamespaceMode_CONTAINER,
+					Ipc:     runtime.NamespaceMode_POD,
+				},
+			},
+		},
+		Labels:      map[string]string{"a": "b"},
+		Annotations: map[string]string{"c": "d"},
+	}
+	metadata := sandboxstore.Metadata{
+		ID:             id,
+		Name:           "test-name",
+		Config:         config,
+		RuntimeHandler: "test-runtime-handler",
+	}
+
+	expected := &runtime.PodSandboxStatus{
+		Id:        id,
+		Metadata:  config.GetMetadata(),
+		CreatedAt: createdAt.UnixNano(),
+		Network: &runtime.PodSandboxNetworkStatus{
+			Ip: ip,
+			AdditionalIps: []*runtime.PodIP{
+				{
+					Ip: additionalIPs[0],
+				},
+				{
+					Ip: additionalIPs[1],
+				},
+			},
+		},
+		Linux: &runtime.LinuxPodSandboxStatus{
+			Namespaces: &runtime.Namespace{
+				Options: &runtime.NamespaceOption{
+					Network: runtime.NamespaceMode_NODE,
+					Pid:     runtime.NamespaceMode_CONTAINER,
+					Ipc:     runtime.NamespaceMode_POD,
+				},
+			},
+		},
+		Labels:         config.GetLabels(),
+		Annotations:    config.GetAnnotations(),
+		RuntimeHandler: "test-runtime-handler",
+	}
+	for desc, test := range map[string]struct {
+		state         sandboxstore.State
+		expectedState runtime.PodSandboxState
+	}{
+		"sandbox state ready": {
+			state:         sandboxstore.StateReady,
+			expectedState: runtime.PodSandboxState_SANDBOX_READY,
+		},
+		"sandbox state not ready": {
+			state:         sandboxstore.StateNotReady,
+			expectedState: runtime.PodSandboxState_SANDBOX_NOTREADY,
+		},
+		"sandbox state unknown": {
+			state:         sandboxstore.StateUnknown,
+			expectedState: runtime.PodSandboxState_SANDBOX_NOTREADY,
+		},
+	} {
+		t.Logf("TestCase: %s", desc)
+		status := sandboxstore.Status{
+			CreatedAt: createdAt,
+			State:     test.state,
+		}
+		expected.State = test.expectedState
+		got := toCRISandboxStatus(metadata, status, ip, additionalIPs)
+		assert.Equal(t, expected, got)
+	}
+}

pkg/cri/server/sandbox_stop.go

@@ -0,0 +1,195 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"syscall"
+	"time"
+
+	eventtypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+// StopPodSandbox stops the sandbox. If there are any running containers in the
+// sandbox, they should be forcibly terminated.
+func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandboxRequest) (*runtime.StopPodSandboxResponse, error) {
+	sandbox, err := c.sandboxStore.Get(r.GetPodSandboxId())
+	if err != nil {
+		return nil, errors.Wrapf(err, "an error occurred when try to find sandbox %q",
+			r.GetPodSandboxId())
+	}
+
+	if err := c.stopPodSandbox(ctx, sandbox); err != nil {
+		return nil, err
+	}
+
+	return &runtime.StopPodSandboxResponse{}, nil
+}
+
+func (c *criService) stopPodSandbox(ctx context.Context, sandbox sandboxstore.Sandbox) error {
+	// Use the full sandbox id.
+	id := sandbox.ID
+
+	// Stop all containers inside the sandbox. This terminates the container forcibly,
+	// and container may still be created, so production should not rely on this behavior.
+	// TODO(random-liu): Introduce a state in sandbox to avoid future container creation.
+	containers := c.containerStore.List()
+	for _, container := range containers {
+		if container.SandboxID != id {
+			continue
+		}
+		// Forcibly stop the container. Do not use `StopContainer`, because it introduces a race
+		// if a container is removed after list.
+		if err := c.stopContainer(ctx, container, 0); err != nil {
+			return errors.Wrapf(err, "failed to stop container %q", container.ID)
+		}
+	}
+
+	if err := c.cleanupSandboxFiles(id, sandbox.Config); err != nil {
+		return errors.Wrap(err, "failed to cleanup sandbox files")
+	}
+
+	// Only stop sandbox container when it's running or unknown.
+	state := sandbox.Status.Get().State
+	if state == sandboxstore.StateReady || state == sandboxstore.StateUnknown {
+		if err := c.stopSandboxContainer(ctx, sandbox); err != nil {
+			return errors.Wrapf(err, "failed to stop sandbox container %q in %q state", id, state)
+		}
+	}
+
+	// Teardown network for sandbox.
+	if sandbox.NetNS != nil {
+		// Use empty netns path if netns is not available. This is defined in:
+		// https://github.com/containernetworking/cni/blob/v0.7.0-alpha1/SPEC.md
+		if closed, err := sandbox.NetNS.Closed(); err != nil {
+			return errors.Wrap(err, "failed to check network namespace closed")
+		} else if closed {
+			sandbox.NetNSPath = ""
+		}
+		if err := c.teardownPodNetwork(ctx, sandbox); err != nil {
+			return errors.Wrapf(err, "failed to destroy network for sandbox %q", id)
+		}
+		if err := sandbox.NetNS.Remove(); err != nil {
+			return errors.Wrapf(err, "failed to remove network namespace for sandbox %q", id)
+		}
+	}
+
+	log.G(ctx).Infof("TearDown network for sandbox %q successfully", id)
+
+	return nil
+}
+
+// stopSandboxContainer kills the sandbox container.
+// `task.Delete` is not called here because it will be called when
+// the event monitor handles the `TaskExit` event.
+func (c *criService) stopSandboxContainer(ctx context.Context, sandbox sandboxstore.Sandbox) error {
+	id := sandbox.ID
+	container := sandbox.Container
+	state := sandbox.Status.Get().State
+	task, err := container.Task(ctx, nil)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return errors.Wrap(err, "failed to get sandbox container")
+		}
+		// Don't return for unknown state, some cleanup needs to be done.
+		if state == sandboxstore.StateUnknown {
+			return cleanupUnknownSandbox(ctx, id, sandbox)
+		}
+		return nil
+	}
+
+	// Handle unknown state.
+	// The cleanup logic is the same with container unknown state.
+	if state == sandboxstore.StateUnknown {
+		// Start an exit handler for containers in unknown state.
+		waitCtx, waitCancel := context.WithCancel(ctrdutil.NamespacedContext())
+		defer waitCancel()
+		exitCh, err := task.Wait(waitCtx)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				return errors.Wrap(err, "failed to wait for task")
+			}
+			return cleanupUnknownSandbox(ctx, id, sandbox)
+		}
+
+		exitCtx, exitCancel := context.WithCancel(context.Background())
+		stopCh := c.eventMonitor.startExitMonitor(exitCtx, id, task.Pid(), exitCh)
+		defer func() {
+			exitCancel()
+			// This ensures that exit monitor is stopped before
+			// `Wait` is cancelled, so no exit event is generated
+			// because of the `Wait` cancellation.
+			<-stopCh
+		}()
+	}
+
+	// Kill the sandbox container.
+	if err = task.Kill(ctx, syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrap(err, "failed to kill sandbox container")
+	}
+
+	return c.waitSandboxStop(ctx, sandbox)
+}
+
+// waitSandboxStop waits for sandbox to be stopped until context is cancelled or
+// the context deadline is exceeded.
+func (c *criService) waitSandboxStop(ctx context.Context, sandbox sandboxstore.Sandbox) error {
+	select {
+	case <-ctx.Done():
+		return errors.Wrapf(ctx.Err(), "wait sandbox container %q", sandbox.ID)
+	case <-sandbox.Stopped():
+		return nil
+	}
+}
+
+// teardownPodNetwork removes the network from the pod
+func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstore.Sandbox) error {
+	if c.netPlugin == nil {
+		return errors.New("cni config not initialized")
+	}
+
+	var (
+		id     = sandbox.ID
+		path   = sandbox.NetNSPath
+		config = sandbox.Config
+	)
+	opts, err := cniNamespaceOpts(id, config)
+	if err != nil {
+		return errors.Wrap(err, "get cni namespace options")
+	}
+
+	return c.netPlugin.Remove(ctx, id, path, opts...)
+}
+
+// cleanupUnknownSandbox cleanup stopped sandbox in unknown state.
+func cleanupUnknownSandbox(ctx context.Context, id string, sandbox sandboxstore.Sandbox) error {
+	// Reuse handleSandboxExit to do the cleanup.
+	return handleSandboxExit(ctx, &eventtypes.TaskExit{
+		ContainerID: id,
+		ID:          id,
+		Pid:         0,
+		ExitStatus:  unknownExitCode,
+		ExitedAt:    time.Now(),
+	}, sandbox)
+}

pkg/cri/server/sandbox_stop_test.go

@@ -0,0 +1,73 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestWaitSandboxStop(t *testing.T) {
+	id := "test-id"
+	for desc, test := range map[string]struct {
+		state     sandboxstore.State
+		cancel    bool
+		timeout   time.Duration
+		expectErr bool
+	}{
+		"should return error if timeout exceeds": {
+			state:     sandboxstore.StateReady,
+			timeout:   200 * time.Millisecond,
+			expectErr: true,
+		},
+		"should return error if context is cancelled": {
+			state:     sandboxstore.StateReady,
+			timeout:   time.Hour,
+			cancel:    true,
+			expectErr: true,
+		},
+		"should not return error if sandbox is stopped before timeout": {
+			state:     sandboxstore.StateNotReady,
+			timeout:   time.Hour,
+			expectErr: false,
+		},
+	} {
+		c := newTestCRIService()
+		sandbox := sandboxstore.NewSandbox(
+			sandboxstore.Metadata{ID: id},
+			sandboxstore.Status{State: test.state},
+		)
+		ctx := context.Background()
+		if test.cancel {
+			cancelledCtx, cancel := context.WithCancel(ctx)
+			cancel()
+			ctx = cancelledCtx
+		}
+		if test.timeout > 0 {
+			timeoutCtx, cancel := context.WithTimeout(ctx, test.timeout)
+			defer cancel()
+			ctx = timeoutCtx
+		}
+		err := c.waitSandboxStop(ctx, sandbox)
+		assert.Equal(t, test.expectErr, err != nil, desc)
+	}
+}

pkg/cri/server/service.go

@@ -0,0 +1,325 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/pkg/streaming"
+	"github.com/containerd/containerd/plugin"
+	cni "github.com/containerd/go-cni"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/cri/store/label"
+
+	"github.com/containerd/containerd/pkg/atomic"
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	snapshotstore "github.com/containerd/containerd/pkg/cri/store/snapshot"
+	osinterface "github.com/containerd/containerd/pkg/os"
+	"github.com/containerd/containerd/pkg/registrar"
+)
+
+// grpcServices are all the grpc services provided by cri containerd.
+type grpcServices interface {
+	runtime.RuntimeServiceServer
+	runtime.ImageServiceServer
+}
+
+// CRIService is the interface implement CRI remote service server.
+type CRIService interface {
+	Run() error
+	// io.Closer is used by containerd to gracefully stop cri service.
+	io.Closer
+	plugin.Service
+	grpcServices
+}
+
+// criService implements CRIService.
+type criService struct {
+	// config contains all configurations.
+	config criconfig.Config
+	// imageFSPath is the path to image filesystem.
+	imageFSPath string
+	// os is an interface for all required os operations.
+	os osinterface.OS
+	// sandboxStore stores all resources associated with sandboxes.
+	sandboxStore *sandboxstore.Store
+	// sandboxNameIndex stores all sandbox names and make sure each name
+	// is unique.
+	sandboxNameIndex *registrar.Registrar
+	// containerStore stores all resources associated with containers.
+	containerStore *containerstore.Store
+	// containerNameIndex stores all container names and make sure each
+	// name is unique.
+	containerNameIndex *registrar.Registrar
+	// imageStore stores all resources associated with images.
+	imageStore *imagestore.Store
+	// snapshotStore stores information of all snapshots.
+	snapshotStore *snapshotstore.Store
+	// netPlugin is used to setup and teardown network when run/stop pod sandbox.
+	netPlugin cni.CNI
+	// client is an instance of the containerd client
+	client *containerd.Client
+	// streamServer is the streaming server serves container streaming request.
+	streamServer streaming.Server
+	// eventMonitor is the monitor monitors containerd events.
+	eventMonitor *eventMonitor
+	// initialized indicates whether the server is initialized. All GRPC services
+	// should return error before the server is initialized.
+	initialized atomic.Bool
+	// cniNetConfMonitor is used to reload cni network conf if there is
+	// any valid fs change events from cni network conf dir.
+	cniNetConfMonitor *cniNetConfSyncer
+	// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
+	baseOCISpecs map[string]*oci.Spec
+}
+
+// NewCRIService returns a new instance of CRIService
+func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIService, error) {
+	var err error
+	labels := label.NewStore()
+	c := &criService{
+		config:             config,
+		client:             client,
+		os:                 osinterface.RealOS{},
+		sandboxStore:       sandboxstore.NewStore(labels),
+		containerStore:     containerstore.NewStore(labels),
+		imageStore:         imagestore.NewStore(client),
+		snapshotStore:      snapshotstore.NewStore(),
+		sandboxNameIndex:   registrar.NewRegistrar(),
+		containerNameIndex: registrar.NewRegistrar(),
+		initialized:        atomic.NewBool(false),
+	}
+
+	if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
+		return nil, errors.Errorf("failed to find snapshotter %q", c.config.ContainerdConfig.Snapshotter)
+	}
+
+	c.imageFSPath = imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter)
+	logrus.Infof("Get image filesystem path %q", c.imageFSPath)
+
+	if err := c.initPlatform(); err != nil {
+		return nil, errors.Wrap(err, "initialize platform")
+	}
+
+	// prepare streaming server
+	c.streamServer, err = newStreamServer(c, config.StreamServerAddress, config.StreamServerPort, config.StreamIdleTimeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create stream server")
+	}
+
+	c.eventMonitor = newEventMonitor(c)
+
+	c.cniNetConfMonitor, err = newCNINetConfSyncer(c.config.NetworkPluginConfDir, c.netPlugin, c.cniLoadOptions())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create cni conf monitor")
+	}
+
+	// Preload base OCI specs
+	c.baseOCISpecs, err = loadBaseOCISpecs(&config)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Register registers all required services onto a specific grpc server.
+// This is used by containerd cri plugin.
+func (c *criService) Register(s *grpc.Server) error {
+	return c.register(s)
+}
+
+// RegisterTCP register all required services onto a GRPC server on TCP.
+// This is used by containerd CRI plugin.
+func (c *criService) RegisterTCP(s *grpc.Server) error {
+	if !c.config.DisableTCPService {
+		return c.register(s)
+	}
+	return nil
+}
+
+// Run starts the CRI service.
+func (c *criService) Run() error {
+	logrus.Info("Start subscribing containerd event")
+	c.eventMonitor.subscribe(c.client)
+
+	logrus.Infof("Start recovering state")
+	if err := c.recover(ctrdutil.NamespacedContext()); err != nil {
+		return errors.Wrap(err, "failed to recover state")
+	}
+
+	// Start event handler.
+	logrus.Info("Start event monitor")
+	eventMonitorErrCh := c.eventMonitor.start()
+
+	// Start snapshot stats syncer, it doesn't need to be stopped.
+	logrus.Info("Start snapshots syncer")
+	snapshotsSyncer := newSnapshotsSyncer(
+		c.snapshotStore,
+		c.client.SnapshotService(c.config.ContainerdConfig.Snapshotter),
+		time.Duration(c.config.StatsCollectPeriod)*time.Second,
+	)
+	snapshotsSyncer.start()
+
+	// Start CNI network conf syncer
+	logrus.Info("Start cni network conf syncer")
+	cniNetConfMonitorErrCh := make(chan error, 1)
+	go func() {
+		defer close(cniNetConfMonitorErrCh)
+		cniNetConfMonitorErrCh <- c.cniNetConfMonitor.syncLoop()
+	}()
+
+	// Start streaming server.
+	logrus.Info("Start streaming server")
+	streamServerErrCh := make(chan error)
+	go func() {
+		defer close(streamServerErrCh)
+		if err := c.streamServer.Start(true); err != nil && err != http.ErrServerClosed {
+			logrus.WithError(err).Error("Failed to start streaming server")
+			streamServerErrCh <- err
+		}
+	}()
+
+	// Set the server as initialized. GRPC services could start serving traffic.
+	c.initialized.Set()
+
+	var eventMonitorErr, streamServerErr, cniNetConfMonitorErr error
+	// Stop the whole CRI service if any of the critical service exits.
+	select {
+	case eventMonitorErr = <-eventMonitorErrCh:
+	case streamServerErr = <-streamServerErrCh:
+	case cniNetConfMonitorErr = <-cniNetConfMonitorErrCh:
+	}
+	if err := c.Close(); err != nil {
+		return errors.Wrap(err, "failed to stop cri service")
+	}
+	// If the error is set above, err from channel must be nil here, because
+	// the channel is supposed to be closed. Or else, we wait and set it.
+	if err := <-eventMonitorErrCh; err != nil {
+		eventMonitorErr = err
+	}
+	logrus.Info("Event monitor stopped")
+	// There is a race condition with http.Server.Serve.
+	// When `Close` is called at the same time with `Serve`, `Close`
+	// may finish first, and `Serve` may still block.
+	// See https://github.com/golang/go/issues/20239.
+	// Here we set a 2 second timeout for the stream server wait,
+	// if it timeout, an error log is generated.
+	// TODO(random-liu): Get rid of this after https://github.com/golang/go/issues/20239
+	// is fixed.
+	const streamServerStopTimeout = 2 * time.Second
+	select {
+	case err := <-streamServerErrCh:
+		if err != nil {
+			streamServerErr = err
+		}
+		logrus.Info("Stream server stopped")
+	case <-time.After(streamServerStopTimeout):
+		logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout)
+	}
+	if eventMonitorErr != nil {
+		return errors.Wrap(eventMonitorErr, "event monitor error")
+	}
+	if streamServerErr != nil {
+		return errors.Wrap(streamServerErr, "stream server error")
+	}
+	if cniNetConfMonitorErr != nil {
+		return errors.Wrap(cniNetConfMonitorErr, "cni network conf monitor error")
+	}
+	return nil
+}
+
+// Close stops the CRI service.
+// TODO(random-liu): Make close synchronous.
+func (c *criService) Close() error {
+	logrus.Info("Stop CRI service")
+	if err := c.cniNetConfMonitor.stop(); err != nil {
+		logrus.WithError(err).Error("failed to stop cni network conf monitor")
+	}
+	c.eventMonitor.stop()
+	if err := c.streamServer.Stop(); err != nil {
+		return errors.Wrap(err, "failed to stop stream server")
+	}
+	return nil
+}
+
+func (c *criService) register(s *grpc.Server) error {
+	instrumented := newInstrumentedService(c)
+	runtime.RegisterRuntimeServiceServer(s, instrumented)
+	runtime.RegisterImageServiceServer(s, instrumented)
+	return nil
+}
+
+// imageFSPath returns containerd image filesystem path.
+// Note that if containerd changes directory layout, we also needs to change this.
+func imageFSPath(rootDir, snapshotter string) string {
+	return filepath.Join(rootDir, fmt.Sprintf("%s.%s", plugin.SnapshotPlugin, snapshotter))
+}
+
+func loadOCISpec(filename string) (*oci.Spec, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open base OCI spec: %s", filename)
+	}
+	defer file.Close()
+
+	spec := oci.Spec{}
+	if err := json.NewDecoder(file).Decode(&spec); err != nil {
+		return nil, errors.Wrap(err, "failed to parse base OCI spec file")
+	}
+
+	return &spec, nil
+}
+
+func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) {
+	specs := map[string]*oci.Spec{}
+	for _, cfg := range config.Runtimes {
+		if cfg.BaseRuntimeSpec == "" {
+			continue
+		}
+
+		// Don't load same file twice
+		if _, ok := specs[cfg.BaseRuntimeSpec]; ok {
+			continue
+		}
+
+		spec, err := loadOCISpec(cfg.BaseRuntimeSpec)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to load base OCI spec from file: %s", cfg.BaseRuntimeSpec)
+		}
+
+		specs[cfg.BaseRuntimeSpec] = spec
+	}
+
+	return specs, nil
+}

pkg/cri/server/service_linux.go

@@ -0,0 +1,70 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/sys"
+	cni "github.com/containerd/go-cni"
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// networkAttachCount is the minimum number of networks the PodSandbox
+// attaches to
+const networkAttachCount = 2
+
+// initPlatform handles linux specific initialization for the CRI service.
+func (c *criService) initPlatform() error {
+	var err error
+
+	if sys.RunningInUserNS() {
+		if !(c.config.DisableCgroup && !c.apparmorEnabled() && c.config.RestrictOOMScoreAdj) {
+			logrus.Warn("Running containerd in a user namespace typically requires disable_cgroup, disable_apparmor, restrict_oom_score_adj set to be true")
+		}
+	}
+
+	if c.config.EnableSelinux {
+		if !selinux.GetEnabled() {
+			logrus.Warn("Selinux is not supported")
+		}
+		if r := c.config.SelinuxCategoryRange; r > 0 {
+			selinux.CategoryRange = uint32(r)
+		}
+	} else {
+		selinux.SetDisabled()
+	}
+
+	// Pod needs to attach to at least loopback network and a non host network,
+	// hence networkAttachCount is 2. If there are more network configs the
+	// pod will be attached to all the networks but we will only use the ip
+	// of the default network interface as the pod IP.
+	c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount),
+		cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
+		cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
+		cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize cni")
+	}
+
+	return nil
+}
+
+// cniLoadOptions returns cni load options for the linux.
+func (c *criService) cniLoadOptions() []cni.CNIOpt {
+	return []cni.CNIOpt{cni.WithLoNetwork, cni.WithDefaultConf}
+}

pkg/cri/server/service_other.go

@@ -0,0 +1,33 @@
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	cni "github.com/containerd/go-cni"
+)
+
+// initPlatform handles linux specific initialization for the CRI service.
+func (c *criService) initPlatform() error {
+	return nil
+}
+
+// cniLoadOptions returns cni load options for the linux.
+func (c *criService) cniLoadOptions() []cni.CNIOpt {
+	return []cni.CNIOpt{}
+}

pkg/cri/server/service_test.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/containerd/containerd/oci"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	servertesting "github.com/containerd/containerd/pkg/cri/server/testing"
+	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+	"github.com/containerd/containerd/pkg/cri/store/label"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	snapshotstore "github.com/containerd/containerd/pkg/cri/store/snapshot"
+	ostesting "github.com/containerd/containerd/pkg/os/testing"
+	"github.com/containerd/containerd/pkg/registrar"
+)
+
+const (
+	testRootDir  = "/test/root"
+	testStateDir = "/test/state"
+	// Use an image id as test sandbox image to avoid image name resolve.
+	// TODO(random-liu): Change this to image name after we have complete image
+	// management unit test framework.
+	testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798"
+	testImageFSPath  = "/test/image/fs/path"
+)
+
+// newTestCRIService creates a fake criService for test.
+func newTestCRIService() *criService {
+	labels := label.NewStore()
+	return &criService{
+		config: criconfig.Config{
+			RootDir:  testRootDir,
+			StateDir: testStateDir,
+			PluginConfig: criconfig.PluginConfig{
+				SandboxImage: testSandboxImage,
+			},
+		},
+		imageFSPath:        testImageFSPath,
+		os:                 ostesting.NewFakeOS(),
+		sandboxStore:       sandboxstore.NewStore(labels),
+		imageStore:         imagestore.NewStore(nil),
+		snapshotStore:      snapshotstore.NewStore(),
+		sandboxNameIndex:   registrar.NewRegistrar(),
+		containerStore:     containerstore.NewStore(labels),
+		containerNameIndex: registrar.NewRegistrar(),
+		netPlugin:          servertesting.NewFakeCNIPlugin(),
+	}
+}
+
+func TestLoadBaseOCISpec(t *testing.T) {
+	spec := oci.Spec{Version: "1.0.2", Hostname: "default"}
+
+	file, err := ioutil.TempFile("", "spec-test-")
+	require.NoError(t, err)
+
+	defer func() {
+		assert.NoError(t, file.Close())
+		assert.NoError(t, os.RemoveAll(file.Name()))
+	}()
+
+	err = json.NewEncoder(file).Encode(&spec)
+	assert.NoError(t, err)
+
+	config := criconfig.Config{}
+	config.Runtimes = map[string]criconfig.Runtime{
+		"runc": {BaseRuntimeSpec: file.Name()},
+	}
+
+	specs, err := loadBaseOCISpecs(&config)
+	assert.NoError(t, err)
+
+	assert.Len(t, specs, 1)
+
+	out, ok := specs[file.Name()]
+	assert.True(t, ok, "expected spec with file name %q", file.Name())
+
+	assert.Equal(t, "1.0.2", out.Version)
+	assert.Equal(t, "default", out.Hostname)
+}

pkg/cri/server/service_windows.go

@@ -0,0 +1,52 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	cni "github.com/containerd/go-cni"
+	"github.com/pkg/errors"
+)
+
+// windowsNetworkAttachCount is the minimum number of networks the PodSandbox
+// attaches to
+const windowsNetworkAttachCount = 1
+
+// initPlatform handles linux specific initialization for the CRI service.
+func (c *criService) initPlatform() error {
+	var err error
+	// For windows, the loopback network is added as default.
+	// There is no need to explicitly add one hence networkAttachCount is 1.
+	// If there are more network configs the pod will be attached to all the
+	// networks but we will only use the ip of the default network interface
+	// as the pod IP.
+	c.netPlugin, err = cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
+		cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
+		cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
+		cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize cni")
+	}
+
+	return nil
+}
+
+// cniLoadOptions returns cni load options for the windows.
+func (c *criService) cniLoadOptions() []cni.CNIOpt {
+	return []cni.CNIOpt{cni.WithDefaultConf}
+}

pkg/cri/server/snapshots.go

@@ -0,0 +1,120 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"time"
+
+	"github.com/containerd/containerd/errdefs"
+	snapshot "github.com/containerd/containerd/snapshots"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	snapshotstore "github.com/containerd/containerd/pkg/cri/store/snapshot"
+)
+
+// snapshotsSyncer syncs snapshot stats periodically. imagefs info and container stats
+// should both use cached result here.
+// TODO(random-liu): Benchmark with high workload. We may need a statsSyncer instead if
+// benchmark result shows that container cpu/memory stats also need to be cached.
+type snapshotsSyncer struct {
+	store       *snapshotstore.Store
+	snapshotter snapshot.Snapshotter
+	syncPeriod  time.Duration
+}
+
+// newSnapshotsSyncer creates a snapshot syncer.
+func newSnapshotsSyncer(store *snapshotstore.Store, snapshotter snapshot.Snapshotter,
+	period time.Duration) *snapshotsSyncer {
+	return &snapshotsSyncer{
+		store:       store,
+		snapshotter: snapshotter,
+		syncPeriod:  period,
+	}
+}
+
+// start starts the snapshots syncer. No stop function is needed because
+// the syncer doesn't update any persistent states, it's fine to let it
+// exit with the process.
+func (s *snapshotsSyncer) start() {
+	tick := time.NewTicker(s.syncPeriod)
+	go func() {
+		defer tick.Stop()
+		// TODO(random-liu): This is expensive. We should do benchmark to
+		// check the resource usage and optimize this.
+		for {
+			if err := s.sync(); err != nil {
+				logrus.WithError(err).Error("Failed to sync snapshot stats")
+			}
+			<-tick.C
+		}
+	}()
+}
+
+// sync updates all snapshots stats.
+func (s *snapshotsSyncer) sync() error {
+	ctx := ctrdutil.NamespacedContext()
+	start := time.Now().UnixNano()
+	var snapshots []snapshot.Info
+	// Do not call `Usage` directly in collect function, because
+	// `Usage` takes time, we don't want `Walk` to hold read lock
+	// of snapshot metadata store for too long time.
+	// TODO(random-liu): Set timeout for the following 2 contexts.
+	if err := s.snapshotter.Walk(ctx, func(ctx context.Context, info snapshot.Info) error {
+		snapshots = append(snapshots, info)
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "walk all snapshots failed")
+	}
+	for _, info := range snapshots {
+		sn, err := s.store.Get(info.Name)
+		if err == nil {
+			// Only update timestamp for non-active snapshot.
+			if sn.Kind == info.Kind && sn.Kind != snapshot.KindActive {
+				sn.Timestamp = time.Now().UnixNano()
+				s.store.Add(sn)
+				continue
+			}
+		}
+		// Get newest stats if the snapshot is new or active.
+		sn = snapshotstore.Snapshot{
+			Key:       info.Name,
+			Kind:      info.Kind,
+			Timestamp: time.Now().UnixNano(),
+		}
+		usage, err := s.snapshotter.Usage(ctx, info.Name)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				logrus.WithError(err).Errorf("Failed to get usage for snapshot %q", info.Name)
+			}
+			continue
+		}
+		sn.Size = uint64(usage.Size)
+		sn.Inodes = uint64(usage.Inodes)
+		s.store.Add(sn)
+	}
+	for _, sn := range s.store.List() {
+		if sn.Timestamp >= start {
+			continue
+		}
+		// Delete the snapshot stats if it's not updated this time.
+		s.store.Delete(sn.Key)
+	}
+	return nil
+}

pkg/cri/server/status.go

@@ -0,0 +1,83 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	goruntime "runtime"
+
+	"github.com/containerd/containerd/log"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// networkNotReadyReason is the reason reported when network is not ready.
+const networkNotReadyReason = "NetworkPluginNotReady"
+
+// Status returns the status of the runtime.
+func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*runtime.StatusResponse, error) {
+	// As a containerd plugin, if CRI plugin is serving request,
+	// containerd must be ready.
+	runtimeCondition := &runtime.RuntimeCondition{
+		Type:   runtime.RuntimeReady,
+		Status: true,
+	}
+	networkCondition := &runtime.RuntimeCondition{
+		Type:   runtime.NetworkReady,
+		Status: true,
+	}
+	// Check the status of the cni initialization
+	if err := c.netPlugin.Status(); err != nil {
+		networkCondition.Status = false
+		networkCondition.Reason = networkNotReadyReason
+		networkCondition.Message = fmt.Sprintf("Network plugin returns error: %v", err)
+	}
+
+	resp := &runtime.StatusResponse{
+		Status: &runtime.RuntimeStatus{Conditions: []*runtime.RuntimeCondition{
+			runtimeCondition,
+			networkCondition,
+		}},
+	}
+	if r.Verbose {
+		configByt, err := json.Marshal(c.config)
+		if err != nil {
+			return nil, err
+		}
+		resp.Info = make(map[string]string)
+		resp.Info["config"] = string(configByt)
+		versionByt, err := json.Marshal(goruntime.Version())
+		if err != nil {
+			return nil, err
+		}
+		resp.Info["golang"] = string(versionByt)
+
+		cniConfig, err := json.Marshal(c.netPlugin.GetConfig())
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to marshal CNI config %v", err)
+		}
+		resp.Info["cniconfig"] = string(cniConfig)
+
+		lastCNILoadStatus := "OK"
+		if lerr := c.cniNetConfMonitor.lastStatus(); lerr != nil {
+			lastCNILoadStatus = lerr.Error()
+		}
+		resp.Info["lastCNILoadStatus"] = lastCNILoadStatus
+	}
+	return resp, nil
+}

pkg/cri/server/streaming.go

@@ -0,0 +1,239 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"math"
+	"net"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	k8snet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/remotecommand"
+	k8scert "k8s.io/client-go/util/cert"
+	"k8s.io/utils/exec"
+
+	ctrdutil "github.com/containerd/containerd/pkg/containerd/util"
+	"github.com/containerd/containerd/pkg/streaming"
+)
+
+type streamListenerMode int
+
+const (
+	x509KeyPairTLS streamListenerMode = iota
+	selfSignTLS
+	withoutTLS
+)
+
+func getStreamListenerMode(c *criService) (streamListenerMode, error) {
+	if c.config.EnableTLSStreaming {
+		if c.config.X509KeyPairStreaming.TLSCertFile != "" && c.config.X509KeyPairStreaming.TLSKeyFile != "" {
+			return x509KeyPairTLS, nil
+		}
+		if c.config.X509KeyPairStreaming.TLSCertFile != "" && c.config.X509KeyPairStreaming.TLSKeyFile == "" {
+			return -1, errors.New("must set X509KeyPairStreaming.TLSKeyFile")
+		}
+		if c.config.X509KeyPairStreaming.TLSCertFile == "" && c.config.X509KeyPairStreaming.TLSKeyFile != "" {
+			return -1, errors.New("must set X509KeyPairStreaming.TLSCertFile")
+		}
+		return selfSignTLS, nil
+	}
+	if c.config.X509KeyPairStreaming.TLSCertFile != "" {
+		return -1, errors.New("X509KeyPairStreaming.TLSCertFile is set but EnableTLSStreaming is not set")
+	}
+	if c.config.X509KeyPairStreaming.TLSKeyFile != "" {
+		return -1, errors.New("X509KeyPairStreaming.TLSKeyFile is set but EnableTLSStreaming is not set")
+	}
+	return withoutTLS, nil
+}
+
+func newStreamServer(c *criService, addr, port, streamIdleTimeout string) (streaming.Server, error) {
+	if addr == "" {
+		a, err := k8snet.ResolveBindAddress(nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get stream server address")
+		}
+		addr = a.String()
+	}
+	config := streaming.DefaultConfig
+	if streamIdleTimeout != "" {
+		var err error
+		config.StreamIdleTimeout, err = time.ParseDuration(streamIdleTimeout)
+		if err != nil {
+			return nil, errors.Wrap(err, "invalid stream idle timeout")
+		}
+	}
+	config.Addr = net.JoinHostPort(addr, port)
+	run := newStreamRuntime(c)
+	tlsMode, err := getStreamListenerMode(c)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid stream server configuration")
+	}
+	switch tlsMode {
+	case x509KeyPairTLS:
+		tlsCert, err := tls.LoadX509KeyPair(c.config.X509KeyPairStreaming.TLSCertFile, c.config.X509KeyPairStreaming.TLSKeyFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to load x509 key pair for stream server")
+		}
+		config.TLSConfig = &tls.Config{
+			Certificates: []tls.Certificate{tlsCert},
+		}
+		return streaming.NewServer(config, run)
+	case selfSignTLS:
+		tlsCert, err := newTLSCert()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to generate tls certificate for stream server")
+		}
+		config.TLSConfig = &tls.Config{
+			Certificates:       []tls.Certificate{tlsCert},
+			InsecureSkipVerify: true,
+		}
+		return streaming.NewServer(config, run)
+	case withoutTLS:
+		return streaming.NewServer(config, run)
+	default:
+		return nil, errors.New("invalid configuration for the stream listener")
+	}
+}
+
+type streamRuntime struct {
+	c *criService
+}
+
+func newStreamRuntime(c *criService) streaming.Runtime {
+	return &streamRuntime{c: c}
+}
+
+// Exec executes a command inside the container. exec.ExitError is returned if the command
+// returns non-zero exit code.
+func (s *streamRuntime) Exec(containerID string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser,
+	tty bool, resize <-chan remotecommand.TerminalSize) error {
+	exitCode, err := s.c.execInContainer(ctrdutil.NamespacedContext(), containerID, execOptions{
+		cmd:    cmd,
+		stdin:  stdin,
+		stdout: stdout,
+		stderr: stderr,
+		tty:    tty,
+		resize: resize,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to exec in container")
+	}
+	if *exitCode == 0 {
+		return nil
+	}
+	return &exec.CodeExitError{
+		Err:  errors.Errorf("error executing command %v, exit code %d", cmd, *exitCode),
+		Code: int(*exitCode),
+	}
+}
+
+func (s *streamRuntime) Attach(containerID string, in io.Reader, out, err io.WriteCloser, tty bool,
+	resize <-chan remotecommand.TerminalSize) error {
+	return s.c.attachContainer(ctrdutil.NamespacedContext(), containerID, in, out, err, tty, resize)
+}
+
+func (s *streamRuntime) PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error {
+	if port <= 0 || port > math.MaxUint16 {
+		return errors.Errorf("invalid port %d", port)
+	}
+	ctx := ctrdutil.NamespacedContext()
+	return s.c.portForward(ctx, podSandboxID, port, stream)
+}
+
+// handleResizing spawns a goroutine that processes the resize channel, calling resizeFunc for each
+// remotecommand.TerminalSize received from the channel.
+func handleResizing(ctx context.Context, resize <-chan remotecommand.TerminalSize, resizeFunc func(size remotecommand.TerminalSize)) {
+	if resize == nil {
+		return
+	}
+
+	go func() {
+		defer runtime.HandleCrash()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case size, ok := <-resize:
+				if !ok {
+					return
+				}
+				if size.Height < 1 || size.Width < 1 {
+					continue
+				}
+				resizeFunc(size)
+			}
+		}
+	}()
+}
+
+// newTLSCert returns a self CA signed tls.certificate.
+// TODO (mikebrow): replace / rewrite this function to support using CA
+// signing of the certificate. Requires a security plan for kubernetes regarding
+// CRI connections / streaming, etc. For example, kubernetes could configure or
+// require a CA service and pass a configuration down through CRI.
+func newTLSCert() (tls.Certificate, error) {
+	fail := func(err error) (tls.Certificate, error) { return tls.Certificate{}, err }
+
+	hostName, err := os.Hostname()
+	if err != nil {
+		return fail(errors.Wrap(err, "failed to get hostname"))
+	}
+
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return fail(errors.Wrap(err, "failed to get host IP addresses"))
+	}
+
+	var alternateIPs []net.IP
+	var alternateDNS []string
+	for _, addr := range addrs {
+		var ip net.IP
+
+		switch v := addr.(type) {
+		case *net.IPNet:
+			ip = v.IP
+		case *net.IPAddr:
+			ip = v.IP
+		default:
+			continue
+		}
+
+		alternateIPs = append(alternateIPs, ip)
+		alternateDNS = append(alternateDNS, ip.String())
+	}
+
+	// Generate a self signed certificate key (CA is self)
+	certPem, keyPem, err := k8scert.GenerateSelfSignedCertKey(hostName, alternateIPs, alternateDNS)
+	if err != nil {
+		return fail(errors.Wrap(err, "certificate key could not be created"))
+	}
+
+	// Load the tls certificate
+	tlsCert, err := tls.X509KeyPair(certPem, keyPem)
+	if err != nil {
+		return fail(errors.Wrap(err, "certificate could not be loaded"))
+	}
+
+	return tlsCert, nil
+}

pkg/cri/server/streaming_test.go

@@ -0,0 +1,153 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"testing"
+
+	"github.com/containerd/containerd/pkg/cri/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateStreamServer(t *testing.T) {
+	for desc, test := range map[string]struct {
+		*criService
+		tlsMode   streamListenerMode
+		expectErr bool
+	}{
+		"should pass with default withoutTLS": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.DefaultConfig(),
+				},
+			},
+			tlsMode:   withoutTLS,
+			expectErr: false,
+		},
+		"should pass with x509KeyPairTLS": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: true,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "non-empty",
+							TLSCertFile: "non-empty",
+						},
+					},
+				},
+			},
+			tlsMode:   x509KeyPairTLS,
+			expectErr: false,
+		},
+		"should pass with selfSign": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: true,
+					},
+				},
+			},
+			tlsMode:   selfSignTLS,
+			expectErr: false,
+		},
+		"should return error with X509 keypair but not EnableTLSStreaming": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: false,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "non-empty",
+							TLSCertFile: "non-empty",
+						},
+					},
+				},
+			},
+			tlsMode:   -1,
+			expectErr: true,
+		},
+		"should return error with X509 TLSCertFile empty": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: true,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "non-empty",
+							TLSCertFile: "",
+						},
+					},
+				},
+			},
+			tlsMode:   -1,
+			expectErr: true,
+		},
+		"should return error with X509 TLSKeyFile empty": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: true,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "",
+							TLSCertFile: "non-empty",
+						},
+					},
+				},
+			},
+			tlsMode:   -1,
+			expectErr: true,
+		},
+		"should return error without EnableTLSStreaming and only TLSCertFile set": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: false,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "",
+							TLSCertFile: "non-empty",
+						},
+					},
+				},
+			},
+			tlsMode:   -1,
+			expectErr: true,
+		},
+		"should return error without EnableTLSStreaming and only TLSKeyFile set": {
+			criService: &criService{
+				config: config.Config{
+					PluginConfig: config.PluginConfig{
+						EnableTLSStreaming: false,
+						X509KeyPairStreaming: config.X509KeyPairStreaming{
+							TLSKeyFile:  "non-empty",
+							TLSCertFile: "",
+						},
+					},
+				},
+			},
+			tlsMode:   -1,
+			expectErr: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			tlsMode, err := getStreamListenerMode(test.criService)
+			if test.expectErr {
+				assert.Error(t, err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, test.tlsMode, tlsMode)
+		})
+	}
+}

pkg/cri/server/testing/fake_cni_plugin.go

@@ -0,0 +1,59 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testing
+
+import (
+	"context"
+
+	cni "github.com/containerd/go-cni"
+)
+
+// FakeCNIPlugin is a fake plugin used for test.
+type FakeCNIPlugin struct {
+	StatusErr error
+	LoadErr   error
+}
+
+// NewFakeCNIPlugin create a FakeCNIPlugin.
+func NewFakeCNIPlugin() *FakeCNIPlugin {
+	return &FakeCNIPlugin{}
+}
+
+// Setup setups the network of PodSandbox.
+func (f *FakeCNIPlugin) Setup(ctx context.Context, id, path string, opts ...cni.NamespaceOpts) (*cni.CNIResult, error) {
+	return nil, nil
+}
+
+// Remove teardown the network of PodSandbox.
+func (f *FakeCNIPlugin) Remove(ctx context.Context, id, path string, opts ...cni.NamespaceOpts) error {
+	return nil
+}
+
+// Status get the status of the plugin.
+func (f *FakeCNIPlugin) Status() error {
+	return f.StatusErr
+}
+
+// Load loads the network config.
+func (f *FakeCNIPlugin) Load(opts ...cni.CNIOpt) error {
+	return f.LoadErr
+}
+
+// GetConfig returns a copy of the CNI plugin configurations as parsed by CNI
+func (f *FakeCNIPlugin) GetConfig() *cni.ConfigResult {
+	return nil
+}

pkg/cri/server/update_runtime_config.go

@@ -0,0 +1,128 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// cniConfigTemplate contains the values containerd will overwrite
+// in the cni config template.
+type cniConfigTemplate struct {
+	// PodCIDR is the cidr for pods on the node.
+	PodCIDR string
+	// PodCIDRRanges is the cidr ranges for pods on the node.
+	PodCIDRRanges []string
+	// Routes is a list of routes configured.
+	Routes []string
+}
+
+const (
+	// cniConfigFileName is the name of cni config file generated by containerd.
+	cniConfigFileName = "10-containerd-net.conflist"
+	// zeroCIDRv6 is the null route for IPv6.
+	zeroCIDRv6 = "::/0"
+	// zeroCIDRv4 is the null route for IPv4.
+	zeroCIDRv4 = "0.0.0.0/0"
+)
+
+// UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
+func (c *criService) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
+	podCIDRs := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
+	if podCIDRs == "" {
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	cidrs := strings.Split(podCIDRs, ",")
+	for i := range cidrs {
+		cidrs[i] = strings.TrimSpace(cidrs[i])
+	}
+	routes, err := getRoutes(cidrs)
+	if err != nil {
+		return nil, errors.Wrap(err, "get routes")
+	}
+
+	confTemplate := c.config.NetworkPluginConfTemplate
+	if confTemplate == "" {
+		log.G(ctx).Info("No cni config template is specified, wait for other system components to drop the config.")
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	if err := c.netPlugin.Status(); err == nil {
+		log.G(ctx).Infof("Network plugin is ready, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	} else if err := c.netPlugin.Load(c.cniLoadOptions()...); err == nil {
+		log.G(ctx).Infof("CNI config is successfully loaded, skip generating cni config from template %q", confTemplate)
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+	log.G(ctx).Infof("Generating cni config from template %q", confTemplate)
+	// generate cni config file from the template with updated pod cidr.
+	t, err := template.ParseFiles(confTemplate)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse cni config template %q", confTemplate)
+	}
+	if err := os.MkdirAll(c.config.NetworkPluginConfDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create cni config directory: %q", c.config.NetworkPluginConfDir)
+	}
+	confFile := filepath.Join(c.config.NetworkPluginConfDir, cniConfigFileName)
+	f, err := os.OpenFile(confFile, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open cni config file %q", confFile)
+	}
+	defer f.Close()
+	if err := t.Execute(f, cniConfigTemplate{
+		PodCIDR:       cidrs[0],
+		PodCIDRRanges: cidrs,
+		Routes:        routes,
+	}); err != nil {
+		return nil, errors.Wrapf(err, "failed to generate cni config file %q", confFile)
+	}
+	return &runtime.UpdateRuntimeConfigResponse{}, nil
+}
+
+// getRoutes generates required routes for the passed in cidrs.
+func getRoutes(cidrs []string) ([]string, error) {
+	var (
+		routes       []string
+		hasV4, hasV6 bool
+	)
+	for _, c := range cidrs {
+		_, cidr, err := net.ParseCIDR(c)
+		if err != nil {
+			return nil, err
+		}
+		if cidr.IP.To4() != nil {
+			hasV4 = true
+		} else {
+			hasV6 = true
+		}
+	}
+	if hasV4 {
+		routes = append(routes, zeroCIDRv4)
+	}
+	if hasV6 {
+		routes = append(routes, zeroCIDRv6)
+	}
+	return routes, nil
+}

pkg/cri/server/update_runtime_config_test.go

@@ -0,0 +1,140 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	servertesting "github.com/containerd/containerd/pkg/cri/server/testing"
+)
+
+func TestUpdateRuntimeConfig(t *testing.T) {
+	const (
+		testTemplate = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "{{.PodCIDR}}",
+			"ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}],
+			"routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}]
+		}
+	},
+	]
+}`
+		testCIDR = "10.0.0.0/24, 2001:4860:4860::/64"
+		expected = `
+{
+	"name": "test-pod-network",
+	"cniVersion": "0.3.1",
+	"plugins": [
+	{
+		"type": "ptp",
+		"mtu": 1460,
+		"ipam": {
+			"type": "host-local",
+			"subnet": "10.0.0.0/24",
+			"ranges": [[{"subnet": "10.0.0.0/24"}], [{"subnet": "2001:4860:4860::/64"}]],
+			"routes": [{"dst": "0.0.0.0/0"}, {"dst": "::/0"}]
+		}
+	},
+	]
+}`
+	)
+
+	for name, test := range map[string]struct {
+		noTemplate      bool
+		emptyCIDR       bool
+		networkReady    bool
+		expectCNIConfig bool
+	}{
+		"should not generate cni config if cidr is empty": {
+			emptyCIDR:       true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if template file is not specified": {
+			noTemplate:      true,
+			expectCNIConfig: false,
+		},
+		"should not generate cni config if network is ready": {
+			networkReady:    true,
+			expectCNIConfig: false,
+		},
+		"should generate cni config if template is specified and cidr is provided": {
+			expectCNIConfig: true,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test-runtime-config")
+			require.NoError(t, err)
+			defer os.RemoveAll(testDir)
+			templateName := filepath.Join(testDir, "template")
+			err = ioutil.WriteFile(templateName, []byte(testTemplate), 0666)
+			require.NoError(t, err)
+			confDir := filepath.Join(testDir, "net.d")
+			confName := filepath.Join(confDir, cniConfigFileName)
+
+			c := newTestCRIService()
+			c.config.CniConfig = criconfig.CniConfig{
+				NetworkPluginConfDir:      confDir,
+				NetworkPluginConfTemplate: templateName,
+			}
+			req := &runtime.UpdateRuntimeConfigRequest{
+				RuntimeConfig: &runtime.RuntimeConfig{
+					NetworkConfig: &runtime.NetworkConfig{
+						PodCidr: testCIDR,
+					},
+				},
+			}
+			if test.noTemplate {
+				c.config.CniConfig.NetworkPluginConfTemplate = ""
+			}
+			if test.emptyCIDR {
+				req.RuntimeConfig.NetworkConfig.PodCidr = ""
+			}
+			if !test.networkReady {
+				c.netPlugin.(*servertesting.FakeCNIPlugin).StatusErr = errors.New("random error")
+				c.netPlugin.(*servertesting.FakeCNIPlugin).LoadErr = errors.New("random error")
+			}
+			_, err = c.UpdateRuntimeConfig(context.Background(), req)
+			assert.NoError(t, err)
+			if !test.expectCNIConfig {
+				_, err := os.Stat(confName)
+				assert.Error(t, err)
+			} else {
+				got, err := ioutil.ReadFile(confName)
+				assert.NoError(t, err)
+				assert.Equal(t, expected, string(got))
+			}
+		})
+	}
+}

pkg/cri/server/version.go

@@ -0,0 +1,42 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"github.com/containerd/containerd/version"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+
+	"github.com/containerd/containerd/pkg/constants"
+)
+
+const (
+	containerName = "containerd"
+	// kubeAPIVersion is the api version of kubernetes.
+	// TODO(random-liu): Change this to actual CRI version.
+	kubeAPIVersion = "0.1.0"
+)
+
+// Version returns the runtime name, runtime version and runtime API version.
+func (c *criService) Version(ctx context.Context, r *runtime.VersionRequest) (*runtime.VersionResponse, error) {
+	return &runtime.VersionResponse{
+		Version:           kubeAPIVersion,
+		RuntimeName:       containerName,
+		RuntimeVersion:    version.Version,
+		RuntimeApiVersion: constants.CRIVersion,
+	}, nil
+}