Merge pull request #7228 from mxpv/sb2

Initial sandbox API CRI integration.

api/next.pb.txt

@@ -4209,9 +4209,45 @@ file {
       type: TYPE_UINT32
       json_name: "pid"
     }
+    field {
+      name: "created_at"
+      number: 3
+      label: LABEL_OPTIONAL
+      type: TYPE_MESSAGE
+      type_name: ".google.protobuf.Timestamp"
+      json_name: "createdAt"
+    }
+    field {
+      name: "labels"
+      number: 4
+      label: LABEL_REPEATED
+      type: TYPE_MESSAGE
+      type_name: ".containerd.services.sandbox.v1.ControllerStartResponse.LabelsEntry"
+      json_name: "labels"
+    }
+    nested_type {
+      name: "LabelsEntry"
+      field {
+        name: "key"
+        number: 1
+        label: LABEL_OPTIONAL
+        type: TYPE_STRING
+        json_name: "key"
+      }
+      field {
+        name: "value"
+        number: 2
+        label: LABEL_OPTIONAL
+        type: TYPE_STRING
+        json_name: "value"
+      }
+      options {
+        map_entry: true
+      }
+    }
   }
   message_type {
-    name: "ControllerShutdownRequest"
+    name: "ControllerStopRequest"
     field {
       name: "sandbox_id"
       number: 1
@@ -4228,7 +4264,7 @@ file {
     }
   }
   message_type {
-    name: "ControllerShutdownResponse"
+    name: "ControllerStopResponse"
   }
   message_type {
     name: "ControllerWaitRequest"
@@ -4315,6 +4351,19 @@ file {
       json_name: "extra"
     }
   }
+  message_type {
+    name: "ControllerDeleteRequest"
+    field {
+      name: "sandbox_id"
+      number: 1
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "sandboxId"
+    }
+  }
+  message_type {
+    name: "ControllerDeleteResponse"
+  }
   service {
     name: "Store"
     method {
@@ -4351,9 +4400,9 @@ file {
       output_type: ".containerd.services.sandbox.v1.ControllerStartResponse"
     }
     method {
-      name: "Shutdown"
+      name: "Stop"
-      input_type: ".containerd.services.sandbox.v1.ControllerShutdownRequest"
+      input_type: ".containerd.services.sandbox.v1.ControllerStopRequest"
-      output_type: ".containerd.services.sandbox.v1.ControllerShutdownResponse"
+      output_type: ".containerd.services.sandbox.v1.ControllerStopResponse"
     }
     method {
       name: "Wait"
@@ -4365,6 +4414,11 @@ file {
       input_type: ".containerd.services.sandbox.v1.ControllerStatusRequest"
       output_type: ".containerd.services.sandbox.v1.ControllerStatusResponse"
     }
+    method {
+      name: "Delete"
+      input_type: ".containerd.services.sandbox.v1.ControllerDeleteRequest"
+      output_type: ".containerd.services.sandbox.v1.ControllerDeleteResponse"
+    }
   }
   options {
     go_package: "github.com/containerd/containerd/api/services/sandbox/v1;sandbox"

api/services/sandbox/v1/sandbox.pb.go

@@ -585,6 +585,8 @@ type ControllerStartResponse struct {
 
 	SandboxID string                 `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"`
 	Pid       uint32                 `protobuf:"varint,2,opt,name=pid,proto3" json:"pid,omitempty"`
+	CreatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"`
+	Labels    map[string]string      `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (x *ControllerStartResponse) Reset() {
@@ -633,7 +635,21 @@ func (x *ControllerStartResponse) GetPid() uint32 {
 	return 0
 }
 
-type ControllerShutdownRequest struct {
+func (x *ControllerStartResponse) GetCreatedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.CreatedAt
+	}
+	return nil
+}
+
+func (x *ControllerStartResponse) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+type ControllerStopRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
@@ -642,8 +658,8 @@ type ControllerShutdownRequest struct {
 	TimeoutSecs uint32 `protobuf:"varint,2,opt,name=timeout_secs,json=timeoutSecs,proto3" json:"timeout_secs,omitempty"`
 }
 
-func (x *ControllerShutdownRequest) Reset() {
+func (x *ControllerStopRequest) Reset() {
-	*x = ControllerShutdownRequest{}
+	*x = ControllerStopRequest{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[12]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -651,13 +667,13 @@ func (x *ControllerShutdownRequest) Reset() {
 	}
 }
 
-func (x *ControllerShutdownRequest) String() string {
+func (x *ControllerStopRequest) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*ControllerShutdownRequest) ProtoMessage() {}
+func (*ControllerStopRequest) ProtoMessage() {}
 
-func (x *ControllerShutdownRequest) ProtoReflect() protoreflect.Message {
+func (x *ControllerStopRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[12]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -669,33 +685,33 @@ func (x *ControllerShutdownRequest) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use ControllerShutdownRequest.ProtoReflect.Descriptor instead.
+// Deprecated: Use ControllerStopRequest.ProtoReflect.Descriptor instead.
-func (*ControllerShutdownRequest) Descriptor() ([]byte, []int) {
+func (*ControllerStopRequest) Descriptor() ([]byte, []int) {
 	return file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{12}
 }
 
-func (x *ControllerShutdownRequest) GetSandboxID() string {
+func (x *ControllerStopRequest) GetSandboxID() string {
 	if x != nil {
 		return x.SandboxID
 	}
 	return ""
 }
 
-func (x *ControllerShutdownRequest) GetTimeoutSecs() uint32 {
+func (x *ControllerStopRequest) GetTimeoutSecs() uint32 {
 	if x != nil {
 		return x.TimeoutSecs
 	}
 	return 0
 }
 
-type ControllerShutdownResponse struct {
+type ControllerStopResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *ControllerShutdownResponse) Reset() {
+func (x *ControllerStopResponse) Reset() {
-	*x = ControllerShutdownResponse{}
+	*x = ControllerStopResponse{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[13]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -703,13 +719,13 @@ func (x *ControllerShutdownResponse) Reset() {
 	}
 }
 
-func (x *ControllerShutdownResponse) String() string {
+func (x *ControllerStopResponse) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*ControllerShutdownResponse) ProtoMessage() {}
+func (*ControllerStopResponse) ProtoMessage() {}
 
-func (x *ControllerShutdownResponse) ProtoReflect() protoreflect.Message {
+func (x *ControllerStopResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[13]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -721,8 +737,8 @@ func (x *ControllerShutdownResponse) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use ControllerShutdownResponse.ProtoReflect.Descriptor instead.
+// Deprecated: Use ControllerStopResponse.ProtoReflect.Descriptor instead.
-func (*ControllerShutdownResponse) Descriptor() ([]byte, []int) {
+func (*ControllerStopResponse) Descriptor() ([]byte, []int) {
 	return file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{13}
 }
 
@@ -962,6 +978,91 @@ func (x *ControllerStatusResponse) GetExtra() *anypb.Any {
 	return nil
 }
 
+type ControllerDeleteRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"`
+}
+
+func (x *ControllerDeleteRequest) Reset() {
+	*x = ControllerDeleteRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[18]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ControllerDeleteRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ControllerDeleteRequest) ProtoMessage() {}
+
+func (x *ControllerDeleteRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[18]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ControllerDeleteRequest.ProtoReflect.Descriptor instead.
+func (*ControllerDeleteRequest) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{18}
+}
+
+func (x *ControllerDeleteRequest) GetSandboxID() string {
+	if x != nil {
+		return x.SandboxID
+	}
+	return ""
+}
+
+type ControllerDeleteResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *ControllerDeleteResponse) Reset() {
+	*x = ControllerDeleteResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[19]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ControllerDeleteResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ControllerDeleteResponse) ProtoMessage() {}
+
+func (x *ControllerDeleteResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[19]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ControllerDeleteResponse.ProtoReflect.Descriptor instead.
+func (*ControllerDeleteResponse) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDescGZIP(), []int{19}
+}
+
 var File_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto protoreflect.FileDescriptor
 
 var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDesc = []byte{
@@ -1032,120 +1133,145 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_
 	0x12, 0x2e, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
 	0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
 	0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
-	0x22, 0x4a, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74,
+	0x22, 0x9d, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53,
-	0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73,
+	0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a,
-	0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69,
+	0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x22, 0x5d, 0x0a, 0x19,
+	0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x39, 0x0a,
-	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f,
+	0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28,
-	0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e,
+	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63,
+	0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x5b, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65,
+	0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+	0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73,
+	0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
+	0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c,
+	0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
+	0x22, 0x59, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74,
+	0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e,
 	0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73,
 	0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65,
 	0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b,
-	0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x73, 0x22, 0x1c, 0x0a, 0x1a, 0x43,
+	0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x43,
-	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77,
+	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73,
-	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, 0x0a, 0x15, 0x43, 0x6f, 0x6e,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
-	0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d,
-	0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64,
+	0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49,
+	0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x72, 0x0a,
-	0x64, 0x22, 0x72, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57,
+	0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52,
-	0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f,
-	0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d,
+	0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78,
-	0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09,
+	0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74,
-	0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
-	0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
+	0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
-	0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69,
+	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41,
-	0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x38, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x74, 0x22, 0x38, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53,
-	0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a,
-	0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01,
+	0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22,
+	0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0xd8, 0x01, 0x0a, 0x18,
-	0xd8, 0x01, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74,
+	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73,
-	0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01,
-	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18,
-	0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x14,
+	0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74,
-	0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73,
+	0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65,
-	0x74, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61,
+	0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18,
-	0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53,
+	0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f,
+	0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05,
-	0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
-	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
-	0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2a,
+	0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2a, 0x0a, 0x05, 0x65, 0x78,
-	0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e,
+	0x74, 0x72, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
-	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52,
-	0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x32, 0xb7, 0x04, 0x0a, 0x05, 0x53,
+	0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x22, 0x38, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
-	0x74, 0x6f, 0x72, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x32,
+	0x6c, 0x6c, 0x65, 0x72, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76,
+	0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18,
-	0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64,
-	0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x22, 0x1a, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x44, 0x65,
-	0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e,
+	0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xb7, 0x04, 0x0a,
-	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78,
+	0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65,
-	0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52,
+	0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74,
+	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76,
-	0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73,
+	0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71,
-	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
-	0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
-	0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64,
-	0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61,
-	0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x44, 0x65,
-	0x6c, 0x65, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
 	0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62,
-	0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74,
+	0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74,
-	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x55, 0x70, 0x64,
-	0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73,
+	0x61, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
-	0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f,
-	0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6b, 0x0a,
+	0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
-	0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
-	0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64,
+	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61,
-	0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74,
+	0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
+	0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06,
+	0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
+	0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e,
+	0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x65, 0x6c,
+	0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
+	0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72,
+	0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x6b, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
 	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61,
 	0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x4c, 0x69,
-	0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x03, 0x47, 0x65,
+	0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74,
-	0x74, 0x12, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73,
+	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e,
-	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e,
+	0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65,
-	0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x03,
-	0x73, 0x74, 0x1a, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e,
+	0x47, 0x65, 0x74, 0x12, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
-	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78,
+	0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f,
-	0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70,
+	0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71,
-	0x6f, 0x6e, 0x73, 0x65, 0x32, 0xfe, 0x03, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
-	0x6c, 0x65, 0x72, 0x12, 0x78, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x36, 0x2e, 0x63,
-	0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
-	0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f,
-	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
 	0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62,
-	0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72,
+	0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65,
-	0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x81, 0x01,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xee, 0x04, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x72,
-	0x0a, 0x08, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x12, 0x39, 0x2e, 0x63, 0x6f, 0x6e,
+	0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x78, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x36,
-	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
+	0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76,
-	0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74,
+	0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e,
-	0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65,
+	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
+	0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e,
+	0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c,
+	0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x75, 0x0a, 0x04, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
+	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61,
+	0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36,
+	0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e,
+	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x04, 0x57, 0x61, 0x69, 0x74, 0x12, 0x35,
+	0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e,
+	0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
 	0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64,
 	0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65,
-	0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7b, 0x0a,
-	0x65, 0x12, 0x75, 0x0a, 0x04, 0x57, 0x61, 0x69, 0x74, 0x12, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74,
+	0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
-	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e,
+	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61,
-	0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72,
+	0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
-	0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65,
+	0x1a, 0x38, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65,
 	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76,
-	0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74,
+	0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74,
+	0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7b, 0x0a, 0x06, 0x44, 0x65,
-	0x75, 0x73, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e,
+	0x6c, 0x65, 0x74, 0x65, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
-	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78,
+	0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62,
-	0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74,
+	0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72,
-	0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x63, 0x6f,
+	0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e,
-	0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
+	0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
-	0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e,
+	0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43,
-	0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73,
+	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75,
-	0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63,
+	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64,
-	0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65,
+	0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f,
-	0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2f, 0x76,
+	0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78,
-	0x31, 0x3b, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2f, 0x76, 0x31, 0x3b, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f,
-	0x33,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -1160,7 +1286,7 @@ func file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto
 	return file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDescData
 }
 
-var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 18)
+var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
 var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_goTypes = []interface{}{
 	(*StoreCreateRequest)(nil),       // 0: containerd.services.sandbox.v1.StoreCreateRequest
 	(*StoreCreateResponse)(nil),      // 1: containerd.services.sandbox.v1.StoreCreateResponse
@@ -1174,52 +1300,59 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_
 	(*StoreGetResponse)(nil),         // 9: containerd.services.sandbox.v1.StoreGetResponse
 	(*ControllerStartRequest)(nil),   // 10: containerd.services.sandbox.v1.ControllerStartRequest
 	(*ControllerStartResponse)(nil),  // 11: containerd.services.sandbox.v1.ControllerStartResponse
-	(*ControllerShutdownRequest)(nil),  // 12: containerd.services.sandbox.v1.ControllerShutdownRequest
+	(*ControllerStopRequest)(nil),    // 12: containerd.services.sandbox.v1.ControllerStopRequest
-	(*ControllerShutdownResponse)(nil), // 13: containerd.services.sandbox.v1.ControllerShutdownResponse
+	(*ControllerStopResponse)(nil),   // 13: containerd.services.sandbox.v1.ControllerStopResponse
 	(*ControllerWaitRequest)(nil),    // 14: containerd.services.sandbox.v1.ControllerWaitRequest
 	(*ControllerWaitResponse)(nil),   // 15: containerd.services.sandbox.v1.ControllerWaitResponse
 	(*ControllerStatusRequest)(nil),  // 16: containerd.services.sandbox.v1.ControllerStatusRequest
 	(*ControllerStatusResponse)(nil), // 17: containerd.services.sandbox.v1.ControllerStatusResponse
-	(*types.Sandbox)(nil),              // 18: containerd.types.Sandbox
+	(*ControllerDeleteRequest)(nil),  // 18: containerd.services.sandbox.v1.ControllerDeleteRequest
-	(*types.Mount)(nil),                // 19: containerd.types.Mount
+	(*ControllerDeleteResponse)(nil), // 19: containerd.services.sandbox.v1.ControllerDeleteResponse
-	(*anypb.Any)(nil),                  // 20: google.protobuf.Any
+	nil,                              // 20: containerd.services.sandbox.v1.ControllerStartResponse.LabelsEntry
-	(*timestamppb.Timestamp)(nil),      // 21: google.protobuf.Timestamp
+	(*types.Sandbox)(nil),            // 21: containerd.types.Sandbox
+	(*types.Mount)(nil),              // 22: containerd.types.Mount
+	(*anypb.Any)(nil),                // 23: google.protobuf.Any
+	(*timestamppb.Timestamp)(nil),    // 24: google.protobuf.Timestamp
 }
 var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_depIdxs = []int32{
-	18, // 0: containerd.services.sandbox.v1.StoreCreateRequest.sandbox:type_name -> containerd.types.Sandbox
+	21, // 0: containerd.services.sandbox.v1.StoreCreateRequest.sandbox:type_name -> containerd.types.Sandbox
-	18, // 1: containerd.services.sandbox.v1.StoreCreateResponse.sandbox:type_name -> containerd.types.Sandbox
+	21, // 1: containerd.services.sandbox.v1.StoreCreateResponse.sandbox:type_name -> containerd.types.Sandbox
-	18, // 2: containerd.services.sandbox.v1.StoreUpdateRequest.sandbox:type_name -> containerd.types.Sandbox
+	21, // 2: containerd.services.sandbox.v1.StoreUpdateRequest.sandbox:type_name -> containerd.types.Sandbox
-	18, // 3: containerd.services.sandbox.v1.StoreUpdateResponse.sandbox:type_name -> containerd.types.Sandbox
+	21, // 3: containerd.services.sandbox.v1.StoreUpdateResponse.sandbox:type_name -> containerd.types.Sandbox
-	18, // 4: containerd.services.sandbox.v1.StoreListResponse.list:type_name -> containerd.types.Sandbox
+	21, // 4: containerd.services.sandbox.v1.StoreListResponse.list:type_name -> containerd.types.Sandbox
-	18, // 5: containerd.services.sandbox.v1.StoreGetResponse.sandbox:type_name -> containerd.types.Sandbox
+	21, // 5: containerd.services.sandbox.v1.StoreGetResponse.sandbox:type_name -> containerd.types.Sandbox
-	19, // 6: containerd.services.sandbox.v1.ControllerStartRequest.rootfs:type_name -> containerd.types.Mount
+	22, // 6: containerd.services.sandbox.v1.ControllerStartRequest.rootfs:type_name -> containerd.types.Mount
-	20, // 7: containerd.services.sandbox.v1.ControllerStartRequest.options:type_name -> google.protobuf.Any
+	23, // 7: containerd.services.sandbox.v1.ControllerStartRequest.options:type_name -> google.protobuf.Any
-	21, // 8: containerd.services.sandbox.v1.ControllerWaitResponse.exited_at:type_name -> google.protobuf.Timestamp
+	24, // 8: containerd.services.sandbox.v1.ControllerStartResponse.created_at:type_name -> google.protobuf.Timestamp
-	21, // 9: containerd.services.sandbox.v1.ControllerStatusResponse.exited_at:type_name -> google.protobuf.Timestamp
+	20, // 9: containerd.services.sandbox.v1.ControllerStartResponse.labels:type_name -> containerd.services.sandbox.v1.ControllerStartResponse.LabelsEntry
-	20, // 10: containerd.services.sandbox.v1.ControllerStatusResponse.extra:type_name -> google.protobuf.Any
+	24, // 10: containerd.services.sandbox.v1.ControllerWaitResponse.exited_at:type_name -> google.protobuf.Timestamp
-	0,  // 11: containerd.services.sandbox.v1.Store.Create:input_type -> containerd.services.sandbox.v1.StoreCreateRequest
+	24, // 11: containerd.services.sandbox.v1.ControllerStatusResponse.exited_at:type_name -> google.protobuf.Timestamp
-	2,  // 12: containerd.services.sandbox.v1.Store.Update:input_type -> containerd.services.sandbox.v1.StoreUpdateRequest
+	23, // 12: containerd.services.sandbox.v1.ControllerStatusResponse.extra:type_name -> google.protobuf.Any
-	4,  // 13: containerd.services.sandbox.v1.Store.Delete:input_type -> containerd.services.sandbox.v1.StoreDeleteRequest
+	0,  // 13: containerd.services.sandbox.v1.Store.Create:input_type -> containerd.services.sandbox.v1.StoreCreateRequest
-	6,  // 14: containerd.services.sandbox.v1.Store.List:input_type -> containerd.services.sandbox.v1.StoreListRequest
+	2,  // 14: containerd.services.sandbox.v1.Store.Update:input_type -> containerd.services.sandbox.v1.StoreUpdateRequest
-	8,  // 15: containerd.services.sandbox.v1.Store.Get:input_type -> containerd.services.sandbox.v1.StoreGetRequest
+	4,  // 15: containerd.services.sandbox.v1.Store.Delete:input_type -> containerd.services.sandbox.v1.StoreDeleteRequest
-	10, // 16: containerd.services.sandbox.v1.Controller.Start:input_type -> containerd.services.sandbox.v1.ControllerStartRequest
+	6,  // 16: containerd.services.sandbox.v1.Store.List:input_type -> containerd.services.sandbox.v1.StoreListRequest
-	12, // 17: containerd.services.sandbox.v1.Controller.Shutdown:input_type -> containerd.services.sandbox.v1.ControllerShutdownRequest
+	8,  // 17: containerd.services.sandbox.v1.Store.Get:input_type -> containerd.services.sandbox.v1.StoreGetRequest
-	14, // 18: containerd.services.sandbox.v1.Controller.Wait:input_type -> containerd.services.sandbox.v1.ControllerWaitRequest
+	10, // 18: containerd.services.sandbox.v1.Controller.Start:input_type -> containerd.services.sandbox.v1.ControllerStartRequest
-	16, // 19: containerd.services.sandbox.v1.Controller.Status:input_type -> containerd.services.sandbox.v1.ControllerStatusRequest
+	12, // 19: containerd.services.sandbox.v1.Controller.Stop:input_type -> containerd.services.sandbox.v1.ControllerStopRequest
-	1,  // 20: containerd.services.sandbox.v1.Store.Create:output_type -> containerd.services.sandbox.v1.StoreCreateResponse
+	14, // 20: containerd.services.sandbox.v1.Controller.Wait:input_type -> containerd.services.sandbox.v1.ControllerWaitRequest
-	3,  // 21: containerd.services.sandbox.v1.Store.Update:output_type -> containerd.services.sandbox.v1.StoreUpdateResponse
+	16, // 21: containerd.services.sandbox.v1.Controller.Status:input_type -> containerd.services.sandbox.v1.ControllerStatusRequest
-	5,  // 22: containerd.services.sandbox.v1.Store.Delete:output_type -> containerd.services.sandbox.v1.StoreDeleteResponse
+	18, // 22: containerd.services.sandbox.v1.Controller.Delete:input_type -> containerd.services.sandbox.v1.ControllerDeleteRequest
-	7,  // 23: containerd.services.sandbox.v1.Store.List:output_type -> containerd.services.sandbox.v1.StoreListResponse
+	1,  // 23: containerd.services.sandbox.v1.Store.Create:output_type -> containerd.services.sandbox.v1.StoreCreateResponse
-	9,  // 24: containerd.services.sandbox.v1.Store.Get:output_type -> containerd.services.sandbox.v1.StoreGetResponse
+	3,  // 24: containerd.services.sandbox.v1.Store.Update:output_type -> containerd.services.sandbox.v1.StoreUpdateResponse
-	11, // 25: containerd.services.sandbox.v1.Controller.Start:output_type -> containerd.services.sandbox.v1.ControllerStartResponse
+	5,  // 25: containerd.services.sandbox.v1.Store.Delete:output_type -> containerd.services.sandbox.v1.StoreDeleteResponse
-	13, // 26: containerd.services.sandbox.v1.Controller.Shutdown:output_type -> containerd.services.sandbox.v1.ControllerShutdownResponse
+	7,  // 26: containerd.services.sandbox.v1.Store.List:output_type -> containerd.services.sandbox.v1.StoreListResponse
-	15, // 27: containerd.services.sandbox.v1.Controller.Wait:output_type -> containerd.services.sandbox.v1.ControllerWaitResponse
+	9,  // 27: containerd.services.sandbox.v1.Store.Get:output_type -> containerd.services.sandbox.v1.StoreGetResponse
-	17, // 28: containerd.services.sandbox.v1.Controller.Status:output_type -> containerd.services.sandbox.v1.ControllerStatusResponse
+	11, // 28: containerd.services.sandbox.v1.Controller.Start:output_type -> containerd.services.sandbox.v1.ControllerStartResponse
-	20, // [20:29] is the sub-list for method output_type
+	13, // 29: containerd.services.sandbox.v1.Controller.Stop:output_type -> containerd.services.sandbox.v1.ControllerStopResponse
-	11, // [11:20] is the sub-list for method input_type
+	15, // 30: containerd.services.sandbox.v1.Controller.Wait:output_type -> containerd.services.sandbox.v1.ControllerWaitResponse
-	11, // [11:11] is the sub-list for extension type_name
+	17, // 31: containerd.services.sandbox.v1.Controller.Status:output_type -> containerd.services.sandbox.v1.ControllerStatusResponse
-	11, // [11:11] is the sub-list for extension extendee
+	19, // 32: containerd.services.sandbox.v1.Controller.Delete:output_type -> containerd.services.sandbox.v1.ControllerDeleteResponse
-	0,  // [0:11] is the sub-list for field type_name
+	23, // [23:33] is the sub-list for method output_type
+	13, // [13:23] is the sub-list for method input_type
+	13, // [13:13] is the sub-list for extension type_name
+	13, // [13:13] is the sub-list for extension extendee
+	0,  // [0:13] is the sub-list for field type_name
 }
 
 func init() { file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_init() }
@@ -1373,7 +1506,7 @@ func file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto
 			}
 		}
 		file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ControllerShutdownRequest); i {
+			switch v := v.(*ControllerStopRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -1385,7 +1518,7 @@ func file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto
 			}
 		}
 		file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*ControllerShutdownResponse); i {
+			switch v := v.(*ControllerStopResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -1444,6 +1577,30 @@ func file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto
 				return nil
 			}
 		}
+		file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ControllerDeleteRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ControllerDeleteResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -1451,7 +1608,7 @@ func file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   18,
+			NumMessages:   21,
 			NumExtensions: 0,
 			NumServices:   2,
 		},

api/services/sandbox/v1/sandbox.proto

@@ -86,9 +86,10 @@ message StoreGetResponse {
 // Controller is an interface to manage runtime sandbox instances.
 service Controller {
 	rpc Start(ControllerStartRequest) returns (ControllerStartResponse);
-	rpc Shutdown(ControllerShutdownRequest) returns (ControllerShutdownResponse);
+	rpc Stop(ControllerStopRequest) returns (ControllerStopResponse);
 	rpc Wait(ControllerWaitRequest) returns (ControllerWaitResponse);
 	rpc Status(ControllerStatusRequest) returns (ControllerStatusResponse);
+	rpc Delete(ControllerDeleteRequest) returns (ControllerDeleteResponse);
 }
 
 message ControllerStartRequest {
@@ -100,14 +101,16 @@ message ControllerStartRequest {
 message ControllerStartResponse {
 	string sandbox_id = 1;
 	uint32 pid = 2;
+	google.protobuf.Timestamp created_at = 3;
+	map<string, string> labels = 4;
 }
 
-message ControllerShutdownRequest {
+message ControllerStopRequest {
 	string sandbox_id = 1;
 	uint32 timeout_secs = 2;
 }
 
-message ControllerShutdownResponse {}
+message ControllerStopResponse {}
 
 message ControllerWaitRequest {
 	string sandbox_id = 1;
@@ -130,3 +133,9 @@ message ControllerStatusResponse {
 	google.protobuf.Timestamp exited_at = 5;
 	google.protobuf.Any extra = 6;
 }
+
+message ControllerDeleteRequest {
+	string sandbox_id = 1;
+}
+
+message ControllerDeleteResponse {}

api/services/sandbox/v1/sandbox_grpc.pb.go

@@ -253,9 +253,10 @@ var Store_ServiceDesc = grpc.ServiceDesc{
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type ControllerClient interface {
 	Start(ctx context.Context, in *ControllerStartRequest, opts ...grpc.CallOption) (*ControllerStartResponse, error)
-	Shutdown(ctx context.Context, in *ControllerShutdownRequest, opts ...grpc.CallOption) (*ControllerShutdownResponse, error)
+	Stop(ctx context.Context, in *ControllerStopRequest, opts ...grpc.CallOption) (*ControllerStopResponse, error)
 	Wait(ctx context.Context, in *ControllerWaitRequest, opts ...grpc.CallOption) (*ControllerWaitResponse, error)
 	Status(ctx context.Context, in *ControllerStatusRequest, opts ...grpc.CallOption) (*ControllerStatusResponse, error)
+	Delete(ctx context.Context, in *ControllerDeleteRequest, opts ...grpc.CallOption) (*ControllerDeleteResponse, error)
 }
 
 type controllerClient struct {
@@ -275,9 +276,9 @@ func (c *controllerClient) Start(ctx context.Context, in *ControllerStartRequest
 	return out, nil
 }
 
-func (c *controllerClient) Shutdown(ctx context.Context, in *ControllerShutdownRequest, opts ...grpc.CallOption) (*ControllerShutdownResponse, error) {
+func (c *controllerClient) Stop(ctx context.Context, in *ControllerStopRequest, opts ...grpc.CallOption) (*ControllerStopResponse, error) {
-	out := new(ControllerShutdownResponse)
+	out := new(ControllerStopResponse)
-	err := c.cc.Invoke(ctx, "/containerd.services.sandbox.v1.Controller/Shutdown", in, out, opts...)
+	err := c.cc.Invoke(ctx, "/containerd.services.sandbox.v1.Controller/Stop", in, out, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -302,14 +303,24 @@ func (c *controllerClient) Status(ctx context.Context, in *ControllerStatusReque
 	return out, nil
 }
 
+func (c *controllerClient) Delete(ctx context.Context, in *ControllerDeleteRequest, opts ...grpc.CallOption) (*ControllerDeleteResponse, error) {
+	out := new(ControllerDeleteResponse)
+	err := c.cc.Invoke(ctx, "/containerd.services.sandbox.v1.Controller/Delete", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ControllerServer is the server API for Controller service.
 // All implementations must embed UnimplementedControllerServer
 // for forward compatibility
 type ControllerServer interface {
 	Start(context.Context, *ControllerStartRequest) (*ControllerStartResponse, error)
-	Shutdown(context.Context, *ControllerShutdownRequest) (*ControllerShutdownResponse, error)
+	Stop(context.Context, *ControllerStopRequest) (*ControllerStopResponse, error)
 	Wait(context.Context, *ControllerWaitRequest) (*ControllerWaitResponse, error)
 	Status(context.Context, *ControllerStatusRequest) (*ControllerStatusResponse, error)
+	Delete(context.Context, *ControllerDeleteRequest) (*ControllerDeleteResponse, error)
 	mustEmbedUnimplementedControllerServer()
 }
 
@@ -320,8 +331,8 @@ type UnimplementedControllerServer struct {
 func (UnimplementedControllerServer) Start(context.Context, *ControllerStartRequest) (*ControllerStartResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Start not implemented")
 }
-func (UnimplementedControllerServer) Shutdown(context.Context, *ControllerShutdownRequest) (*ControllerShutdownResponse, error) {
+func (UnimplementedControllerServer) Stop(context.Context, *ControllerStopRequest) (*ControllerStopResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Shutdown not implemented")
+	return nil, status.Errorf(codes.Unimplemented, "method Stop not implemented")
 }
 func (UnimplementedControllerServer) Wait(context.Context, *ControllerWaitRequest) (*ControllerWaitResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Wait not implemented")
@@ -329,6 +340,9 @@ func (UnimplementedControllerServer) Wait(context.Context, *ControllerWaitReques
 func (UnimplementedControllerServer) Status(context.Context, *ControllerStatusRequest) (*ControllerStatusResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Status not implemented")
 }
+func (UnimplementedControllerServer) Delete(context.Context, *ControllerDeleteRequest) (*ControllerDeleteResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented")
+}
 func (UnimplementedControllerServer) mustEmbedUnimplementedControllerServer() {}
 
 // UnsafeControllerServer may be embedded to opt out of forward compatibility for this service.
@@ -360,20 +374,20 @@ func _Controller_Start_Handler(srv interface{}, ctx context.Context, dec func(in
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Controller_Shutdown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+func _Controller_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(ControllerShutdownRequest)
+	in := new(ControllerStopRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(ControllerServer).Shutdown(ctx, in)
+		return srv.(ControllerServer).Stop(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
 		Server:     srv,
-		FullMethod: "/containerd.services.sandbox.v1.Controller/Shutdown",
+		FullMethod: "/containerd.services.sandbox.v1.Controller/Stop",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ControllerServer).Shutdown(ctx, req.(*ControllerShutdownRequest))
+		return srv.(ControllerServer).Stop(ctx, req.(*ControllerStopRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
@@ -414,6 +428,24 @@ func _Controller_Status_Handler(srv interface{}, ctx context.Context, dec func(i
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Controller_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ControllerDeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControllerServer).Delete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/containerd.services.sandbox.v1.Controller/Delete",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControllerServer).Delete(ctx, req.(*ControllerDeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // Controller_ServiceDesc is the grpc.ServiceDesc for Controller service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -426,8 +458,8 @@ var Controller_ServiceDesc = grpc.ServiceDesc{
 			Handler:    _Controller_Start_Handler,
 		},
 		{
-			MethodName: "Shutdown",
+			MethodName: "Stop",
-			Handler:    _Controller_Shutdown_Handler,
+			Handler:    _Controller_Stop_Handler,
 		},
 		{
 			MethodName: "Wait",
@@ -437,6 +469,10 @@ var Controller_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "Status",
 			Handler:    _Controller_Status_Handler,
 		},
+		{
+			MethodName: "Delete",
+			Handler:    _Controller_Delete_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "github.com/containerd/containerd/api/services/sandbox/v1/sandbox.proto",

integration/build_local_containerd_helper_test_linux.go

@@ -19,5 +19,6 @@ package integration
 import (
 	// Register for linux platforms
 	_ "github.com/containerd/containerd/runtime/v1/linux"
+	_ "github.com/containerd/containerd/services/sandbox" // WithInMemoryServices will fail otherwise
 	_ "github.com/containerd/containerd/snapshots/overlay/plugin"
 )

metadata/sandbox.go

@@ -369,9 +369,5 @@ func (s *sandboxStore) validate(new *api.Sandbox) error {
 		return fmt.Errorf("updated date must not be zero: %w", errdefs.ErrInvalidArgument)
 	}
 
-	if new.Runtime.Name == "" {
-		return fmt.Errorf("sandbox.Runtime.Name must be set: %w", errdefs.ErrInvalidArgument)
-	}
-
 	return nil
 }

pkg/cri/sbserver/helpers.go

@@ -46,6 +46,8 @@ import (
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
+// TODO: Move common helpers for sbserver and podsandbox to a dedicated package once basic services are functinal.
+
 const (
 	// errorStartReason is the exit reason when fails to start container.
 	errorStartReason = "StartError"
@@ -219,9 +221,9 @@ func getUserFromImage(user string) (*int64, string) {
 	return &uid, ""
 }
 
-// ensureImageExists returns corresponding metadata of the image reference, if image is not
+// EnsureImageExists returns corresponding metadata of the image reference, if image is not
 // pulled yet, the function will pull the image.
-func (c *criService) ensureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig) (*imagestore.Image, error) {
+func (c *criService) EnsureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig) (*imagestore.Image, error) {
 	image, err := c.localResolve(ref)
 	if err != nil && !errdefs.IsNotFound(err) {
 		return nil, fmt.Errorf("failed to get image %q: %w", ref, err)

pkg/cri/sbserver/helpers_linux.go

@@ -35,16 +35,11 @@ import (
 	"github.com/containerd/containerd/pkg/seutil"
 	"github.com/moby/sys/mountinfo"
 	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/selinux/go-selinux/label"
 	"golang.org/x/sys/unix"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
 const (
-	// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
-	defaultSandboxOOMAdj = -998
-	// defaultShmSize is the default size of the sandbox shm.
-	defaultShmSize = int64(1024 * 1024 * 64)
 	// relativeRootfsPath is the rootfs path relative to bundle path.
 	relativeRootfsPath = "rootfs"
 	// devShm is the default path of /dev/shm.
@@ -115,14 +110,6 @@ func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
 	return labels, nil
 }
 
-func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
-	labels, err := toLabel(selinuxOpts)
-	if err != nil {
-		return "", "", err
-	}
-	return label.InitLabels(labels)
-}
-
 func checkSelinuxLevel(level string) error {
 	if len(level) == 0 {
 		return nil

pkg/cri/sbserver/podsandbox/container_linux.go

@@ -0,0 +1,148 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// TODO: these are copied from container_create_linux.go and should be consolidated later.
+
+package podsandbox
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/containerd/contrib/seccomp"
+	"github.com/containerd/containerd/oci"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const (
+	// profileNamePrefix is the prefix for loading profiles on a localhost. Eg. AppArmor localhost/profileName.
+	profileNamePrefix = "localhost/" // TODO (mikebrow): get localhost/ & runtime/default from CRI kubernetes/kubernetes#51747
+	// runtimeDefault indicates that we should use or create a runtime default profile.
+	runtimeDefault = "runtime/default"
+	// dockerDefault indicates that we should use or create a docker default profile.
+	dockerDefault = "docker/default"
+	// unconfinedProfile is a string indicating one should run a pod/containerd without a security profile
+	unconfinedProfile = "unconfined"
+)
+
+// generateSeccompSpecOpts generates containerd SpecOpts for seccomp.
+func (c *Controller) generateSeccompSpecOpts(sp *runtime.SecurityProfile, privileged, seccompEnabled bool) (oci.SpecOpts, error) {
+	if privileged {
+		// Do not set seccomp profile when container is privileged
+		return nil, nil
+	}
+	if !seccompEnabled {
+		if sp != nil {
+			if sp.ProfileType != runtime.SecurityProfile_Unconfined {
+				return nil, errors.New("seccomp is not supported")
+			}
+		}
+		return nil, nil
+	}
+
+	if sp == nil {
+		return nil, nil
+	}
+
+	if sp.ProfileType != runtime.SecurityProfile_Localhost && sp.LocalhostRef != "" {
+		return nil, errors.New("seccomp config invalid LocalhostRef must only be set if ProfileType is Localhost")
+	}
+	switch sp.ProfileType {
+	case runtime.SecurityProfile_Unconfined:
+		// Do not set seccomp profile.
+		return nil, nil
+	case runtime.SecurityProfile_RuntimeDefault:
+		return seccomp.WithDefaultProfile(), nil
+	case runtime.SecurityProfile_Localhost:
+		// trimming the localhost/ prefix just in case even though it should not
+		// be necessary with the new SecurityProfile struct
+		return seccomp.WithProfile(strings.TrimPrefix(sp.LocalhostRef, profileNamePrefix)), nil
+	default:
+		return nil, errors.New("seccomp unknown ProfileType")
+	}
+}
+
+func generateSeccompSecurityProfile(profilePath string, unsetProfilePath string) (*runtime.SecurityProfile, error) {
+	if profilePath != "" {
+		return generateSecurityProfile(profilePath)
+	}
+	if unsetProfilePath != "" {
+		return generateSecurityProfile(unsetProfilePath)
+	}
+	return nil, nil
+}
+
+func generateSecurityProfile(profilePath string) (*runtime.SecurityProfile, error) {
+	switch profilePath {
+	case runtimeDefault, dockerDefault, "":
+		return &runtime.SecurityProfile{
+			ProfileType: runtime.SecurityProfile_RuntimeDefault,
+		}, nil
+	case unconfinedProfile:
+		return &runtime.SecurityProfile{
+			ProfileType: runtime.SecurityProfile_Unconfined,
+		}, nil
+	default:
+		// Require and Trim default profile name prefix
+		if !strings.HasPrefix(profilePath, profileNamePrefix) {
+			return nil, fmt.Errorf("invalid profile %q", profilePath)
+		}
+		return &runtime.SecurityProfile{
+			ProfileType:  runtime.SecurityProfile_Localhost,
+			LocalhostRef: strings.TrimPrefix(profilePath, profileNamePrefix),
+		}, nil
+	}
+}
+
+// generateUserString generates valid user string based on OCI Image Spec
+// v1.0.0.
+//
+// CRI defines that the following combinations are valid:
+//
+// (none) -> ""
+// username -> username
+// username, uid -> username
+// username, uid, gid -> username:gid
+// username, gid -> username:gid
+// uid -> uid
+// uid, gid -> uid:gid
+// gid -> error
+//
+// TODO(random-liu): Add group name support in CRI.
+func generateUserString(username string, uid, gid *runtime.Int64Value) (string, error) {
+	var userstr, groupstr string
+	if uid != nil {
+		userstr = strconv.FormatInt(uid.GetValue(), 10)
+	}
+	if username != "" {
+		userstr = username
+	}
+	if gid != nil {
+		groupstr = strconv.FormatInt(gid.GetValue(), 10)
+	}
+	if userstr == "" {
+		if groupstr != "" {
+			return "", fmt.Errorf("user group %q is specified without user", groupstr)
+		}
+		return "", nil
+	}
+	if groupstr != "" {
+		userstr = userstr + ":" + groupstr
+	}
+	return userstr, nil
+}

pkg/cri/sbserver/podsandbox/controller.go

@@ -0,0 +1,93 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	api "github.com/containerd/containerd/api/services/sandbox/v1"
+	"github.com/containerd/containerd/oci"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	osinterface "github.com/containerd/containerd/pkg/os"
+	"github.com/containerd/containerd/sandbox"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+// CRIService interface contains things required by controller, but not yet refactored from criService.
+// TODO: this will be removed in subsequent iterations.
+type CRIService interface {
+	EnsureImageExists(ctx context.Context, ref string, config *runtime.PodSandboxConfig) (*imagestore.Image, error)
+
+	// TODO: we should implement Controller.Wait and use it instead of this to monitor sandbox exit.
+	StartSandboxExitMonitor(ctx context.Context, id string, pid uint32, exitCh <-chan containerd.ExitStatus) <-chan struct{}
+}
+
+type Controller struct {
+	// config contains all configurations.
+	config criconfig.Config
+	// client is an instance of the containerd client
+	client *containerd.Client
+	// sandboxStore stores all resources associated with sandboxes.
+	sandboxStore *sandboxstore.Store
+	// os is an interface for all required os operations.
+	os osinterface.OS
+	// cri is CRI service that provides missing gaps needed by controller.
+	cri CRIService
+	// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
+	baseOCISpecs map[string]*oci.Spec
+}
+
+func New(
+	config criconfig.Config,
+	client *containerd.Client,
+	sandboxStore *sandboxstore.Store,
+	os osinterface.OS,
+	cri CRIService,
+	baseOCISpecs map[string]*oci.Spec,
+) *Controller {
+	return &Controller{
+		config:       config,
+		client:       client,
+		sandboxStore: sandboxStore,
+		os:           os,
+		cri:          cri,
+		baseOCISpecs: baseOCISpecs,
+	}
+}
+
+var _ sandbox.Controller = (*Controller)(nil)
+
+func (c *Controller) Stop(ctx context.Context, sandboxID string) (*api.ControllerStopResponse, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (c *Controller) Delete(ctx context.Context, sandboxID string) (*api.ControllerDeleteResponse, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (c *Controller) Wait(ctx context.Context, sandboxID string) (*api.ControllerWaitResponse, error) {
+	panic("implement me")
+}
+
+func (c *Controller) Status(ctx context.Context, sandboxID string) (*api.ControllerStatusResponse, error) {
+	panic("implement me")
+}

pkg/cri/sbserver/podsandbox/controller_test.go

@@ -0,0 +1,52 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	"github.com/containerd/containerd/pkg/cri/store/label"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	ostesting "github.com/containerd/containerd/pkg/os/testing"
+)
+
+const (
+	testRootDir  = "/test/root"
+	testStateDir = "/test/state"
+	// Use an image id as test sandbox image to avoid image name resolve.
+	// TODO(random-liu): Change this to image name after we have complete image
+	// management unit test framework.
+	testSandboxImage = "sha256:c75bebcdd211f41b3a460c7bf82970ed6c75acaab9cd4c9a4e125b03ca113798"
+)
+
+var testConfig = criconfig.Config{
+	RootDir:  testRootDir,
+	StateDir: testStateDir,
+	PluginConfig: criconfig.PluginConfig{
+		SandboxImage:                     testSandboxImage,
+		TolerateMissingHugetlbController: true,
+	},
+}
+
+// newControllerService creates a fake criService for test.
+func newControllerService() *Controller {
+	labels := label.NewStore()
+	return &Controller{
+		config:       testConfig,
+		os:           ostesting.NewFakeOS(),
+		sandboxStore: sandboxstore.NewStore(labels),
+	}
+}

pkg/cri/sbserver/podsandbox/helpers.go

@@ -0,0 +1,277 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/containers"
+	clabels "github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/oci"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	imagestore "github.com/containerd/containerd/pkg/cri/store/image"
+	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
+	runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/typeurl"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+
+	runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
+	imagedigest "github.com/opencontainers/go-digest"
+	"github.com/pelletier/go-toml"
+)
+
+const (
+
+	// sandboxesDir contains all sandbox root. A sandbox root is the running
+	// directory of the sandbox, all files created for the sandbox will be
+	// placed under this directory.
+	sandboxesDir = "sandboxes"
+	// criContainerdPrefix is common prefix for cri-containerd
+	criContainerdPrefix = "io.cri-containerd"
+	// containerKindLabel is a label key indicating container is sandbox container or application container
+	containerKindLabel = criContainerdPrefix + ".kind"
+	// containerKindSandbox is a label value indicating container is sandbox container
+	containerKindSandbox = "sandbox"
+	// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
+	sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
+	// runtimeRunhcsV1 is the runtime type for runhcs.
+	runtimeRunhcsV1 = "io.containerd.runhcs.v1"
+)
+
+// getSandboxRootDir returns the root directory for managing sandbox files,
+// e.g. hosts files.
+func (c *Controller) getSandboxRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, sandboxesDir, id)
+}
+
+// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
+// e.g. named pipes.
+func (c *Controller) getVolatileSandboxRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, sandboxesDir, id)
+}
+
+// getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference.
+func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) {
+	var repoTag, repoDigest string
+	if _, ok := namedRef.(docker.NamedTagged); ok {
+		repoTag = namedRef.String()
+	}
+	if _, ok := namedRef.(docker.Canonical); ok {
+		repoDigest = namedRef.String()
+	} else if !schema1 {
+		// digest is not actual repo digest for schema1 image.
+		repoDigest = namedRef.Name() + "@" + digest.String()
+	}
+	return repoDigest, repoTag
+}
+
+// toContainerdImage converts an image object in image store to containerd image handler.
+func (c *Controller) toContainerdImage(ctx context.Context, image imagestore.Image) (containerd.Image, error) {
+	// image should always have at least one reference.
+	if len(image.References) == 0 {
+		return nil, fmt.Errorf("invalid image with no reference %q", image.ID)
+	}
+	return c.client.GetImage(ctx, image.References[0])
+}
+
+// getUserFromImage gets uid or user name of the image user.
+// If user is numeric, it will be treated as uid; or else, it is treated as user name.
+func getUserFromImage(user string) (*int64, string) {
+	// return both empty if user is not specified in the image.
+	if user == "" {
+		return nil, ""
+	}
+	// split instances where the id may contain user:group
+	user = strings.Split(user, ":")[0]
+	// user could be either uid or user name. Try to interpret as numeric uid.
+	uid, err := strconv.ParseInt(user, 10, 64)
+	if err != nil {
+		// If user is non numeric, assume it's user name.
+		return nil, user
+	}
+	// If user is a numeric uid.
+	return &uid, ""
+}
+
+// buildLabel builds the labels from config to be passed to containerd
+func buildLabels(configLabels, imageConfigLabels map[string]string, containerType string) map[string]string {
+	labels := make(map[string]string)
+
+	for k, v := range imageConfigLabels {
+		if err := clabels.Validate(k, v); err == nil {
+			labels[k] = v
+		} else {
+			// In case the image label is invalid, we output a warning and skip adding it to the
+			// container.
+			logrus.WithError(err).Warnf("unable to add image label with key %s to the container", k)
+		}
+	}
+	// labels from the CRI request (config) will override labels in the image config
+	for k, v := range configLabels {
+		labels[k] = v
+	}
+	labels[containerKindLabel] = containerType
+	return labels
+}
+
+// parseImageReferences parses a list of arbitrary image references and returns
+// the repotags and repodigests
+func parseImageReferences(refs []string) ([]string, []string) {
+	var tags, digests []string
+	for _, ref := range refs {
+		parsed, err := docker.ParseAnyReference(ref)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsed.(docker.Canonical); ok {
+			digests = append(digests, parsed.String())
+		} else if _, ok := parsed.(docker.Tagged); ok {
+			tags = append(tags, parsed.String())
+		}
+	}
+	return tags, digests
+}
+
+// generateRuntimeOptions generates runtime options from cri plugin config.
+func generateRuntimeOptions(r criconfig.Runtime, c criconfig.Config) (interface{}, error) {
+	if r.Options == nil {
+		if r.Type != plugin.RuntimeLinuxV1 {
+			return nil, nil
+		}
+		// This is a legacy config, generate runctypes.RuncOptions.
+		return &runctypes.RuncOptions{
+			Runtime:       r.Engine,
+			RuntimeRoot:   r.Root,
+			SystemdCgroup: c.SystemdCgroup,
+		}, nil
+	}
+	optionsTree, err := toml.TreeFromMap(r.Options)
+	if err != nil {
+		return nil, err
+	}
+	options := getRuntimeOptionsType(r.Type)
+	if err := optionsTree.Unmarshal(options); err != nil {
+		return nil, err
+	}
+	return options, nil
+}
+
+// getRuntimeOptionsType gets empty runtime options by the runtime type name.
+func getRuntimeOptionsType(t string) interface{} {
+	switch t {
+	case plugin.RuntimeRuncV1:
+		fallthrough
+	case plugin.RuntimeRuncV2:
+		return &runcoptions.Options{}
+	case plugin.RuntimeLinuxV1:
+		return &runctypes.RuncOptions{}
+	case runtimeRunhcsV1:
+		return &runhcsoptions.Options{}
+	default:
+		return &runtimeoptions.Options{}
+	}
+}
+
+// getRuntimeOptions get runtime options from container metadata.
+func getRuntimeOptions(c containers.Container) (interface{}, error) {
+	from := c.Runtime.Options
+	if from == nil || from.GetValue() == nil {
+		return nil, nil
+	}
+	opts, err := typeurl.UnmarshalAny(from)
+	if err != nil {
+		return nil, err
+	}
+	return opts, nil
+}
+
+// getPassthroughAnnotations filters requested pod annotations by comparing
+// against permitted annotations for the given runtime.
+func getPassthroughAnnotations(podAnnotations map[string]string,
+	runtimePodAnnotations []string) (passthroughAnnotations map[string]string) {
+	passthroughAnnotations = make(map[string]string)
+
+	for podAnnotationKey, podAnnotationValue := range podAnnotations {
+		for _, pattern := range runtimePodAnnotations {
+			// Use path.Match instead of filepath.Match here.
+			// filepath.Match treated `\\` as path separator
+			// on windows, which is not what we want.
+			if ok, _ := path.Match(pattern, podAnnotationKey); ok {
+				passthroughAnnotations[podAnnotationKey] = podAnnotationValue
+			}
+		}
+	}
+	return passthroughAnnotations
+}
+
+// runtimeSpec returns a default runtime spec used in cri-containerd.
+func (c *Controller) runtimeSpec(id string, baseSpecFile string, opts ...oci.SpecOpts) (*runtimespec.Spec, error) {
+	// GenerateSpec needs namespace.
+	ctx := ctrdutil.NamespacedContext()
+	container := &containers.Container{ID: id}
+
+	if baseSpecFile != "" {
+		baseSpec, ok := c.baseOCISpecs[baseSpecFile]
+		if !ok {
+			return nil, fmt.Errorf("can't find base OCI spec %q", baseSpecFile)
+		}
+
+		spec := oci.Spec{}
+		if err := ctrdutil.DeepCopy(&spec, &baseSpec); err != nil {
+			return nil, fmt.Errorf("failed to clone OCI spec: %w", err)
+		}
+
+		// Fix up cgroups path
+		applyOpts := append([]oci.SpecOpts{oci.WithNamespacedCgroup()}, opts...)
+
+		if err := oci.ApplyOpts(ctx, nil, container, &spec, applyOpts...); err != nil {
+			return nil, fmt.Errorf("failed to apply OCI options: %w", err)
+		}
+
+		return &spec, nil
+	}
+
+	spec, err := oci.GenerateSpec(ctx, nil, container, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate spec: %w", err)
+	}
+
+	return spec, nil
+}
+
+// Overrides the default snapshotter if Snapshotter is set for this runtime.
+// See See https://github.com/containerd/containerd/issues/6657
+func (c *Controller) runtimeSnapshotter(ctx context.Context, ociRuntime criconfig.Runtime) string {
+	if ociRuntime.Snapshotter == "" {
+		return c.config.ContainerdConfig.Snapshotter
+	}
+
+	log.G(ctx).Debugf("Set snapshotter for runtime %s to %s", ociRuntime.Type, ociRuntime.Snapshotter)
+	return ociRuntime.Snapshotter
+}

pkg/cri/sbserver/podsandbox/helpers_linux.go

@@ -0,0 +1,255 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/pkg/seccomp"
+	"github.com/containerd/containerd/pkg/seutil"
+	"github.com/moby/sys/mountinfo"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const (
+	// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
+	defaultSandboxOOMAdj = -998
+	// defaultShmSize is the default size of the sandbox shm.
+	defaultShmSize = int64(1024 * 1024 * 64)
+	// relativeRootfsPath is the rootfs path relative to bundle path.
+	relativeRootfsPath = "rootfs"
+	// devShm is the default path of /dev/shm.
+	devShm = "/dev/shm"
+	// etcHosts is the default path of /etc/hosts file.
+	etcHosts = "/etc/hosts"
+	// resolvConfPath is the abs path of resolv.conf on host or container.
+	resolvConfPath = "/etc/resolv.conf"
+)
+
+// getCgroupsPath generates container cgroups path.
+func getCgroupsPath(cgroupsParent, id string) string {
+	base := path.Base(cgroupsParent)
+	if strings.HasSuffix(base, ".slice") {
+		// For a.slice/b.slice/c.slice, base is c.slice.
+		// runc systemd cgroup path format is "slice:prefix:name".
+		return strings.Join([]string{base, "cri-containerd", id}, ":")
+	}
+	return filepath.Join(cgroupsParent, id)
+}
+
+// getSandboxHostname returns the hostname file path inside the sandbox root directory.
+func (c *Controller) getSandboxHostname(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hostname")
+}
+
+// getSandboxHosts returns the hosts file path inside the sandbox root directory.
+func (c *Controller) getSandboxHosts(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "hosts")
+}
+
+// getResolvPath returns resolv.conf filepath for specified sandbox.
+func (c *Controller) getResolvPath(id string) string {
+	return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
+}
+
+// getSandboxDevShm returns the shm file path inside the sandbox root directory.
+func (c *Controller) getSandboxDevShm(id string) string {
+	return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
+}
+
+func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
+	var labels []string
+
+	if selinuxOptions == nil {
+		return nil, nil
+	}
+	if err := checkSelinuxLevel(selinuxOptions.Level); err != nil {
+		return nil, err
+	}
+	if selinuxOptions.User != "" {
+		labels = append(labels, "user:"+selinuxOptions.User)
+	}
+	if selinuxOptions.Role != "" {
+		labels = append(labels, "role:"+selinuxOptions.Role)
+	}
+	if selinuxOptions.Type != "" {
+		labels = append(labels, "type:"+selinuxOptions.Type)
+	}
+	if selinuxOptions.Level != "" {
+		labels = append(labels, "level:"+selinuxOptions.Level)
+	}
+
+	return labels, nil
+}
+
+func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
+	labels, err := toLabel(selinuxOpts)
+	if err != nil {
+		return "", "", err
+	}
+	return label.InitLabels(labels)
+}
+
+func checkSelinuxLevel(level string) error {
+	if len(level) == 0 {
+		return nil
+	}
+
+	matched, err := regexp.MatchString(`^s\d(-s\d)??(:c\d{1,4}(\.c\d{1,4})?(,c\d{1,4}(\.c\d{1,4})?)*)?$`, level)
+	if err != nil {
+		return fmt.Errorf("the format of 'level' %q is not correct: %w", level, err)
+	}
+	if !matched {
+		return fmt.Errorf("the format of 'level' %q is not correct", level)
+	}
+	return nil
+}
+
+func (c *Controller) seccompEnabled() bool {
+	return seccomp.IsEnabled()
+}
+
+// unmountRecursive unmounts the target and all mounts underneath, starting with
+// the deepest mount first.
+func unmountRecursive(ctx context.Context, target string) error {
+	toUnmount, err := mountinfo.GetMounts(mountinfo.PrefixFilter(target))
+	if err != nil {
+		return err
+	}
+
+	// Make the deepest mount be first
+	sort.Slice(toUnmount, func(i, j int) bool {
+		return len(toUnmount[i].Mountpoint) > len(toUnmount[j].Mountpoint)
+	})
+
+	for i, m := range toUnmount {
+		if err := mount.UnmountAll(m.Mountpoint, unix.MNT_DETACH); err != nil {
+			if i == len(toUnmount)-1 { // last mount
+				return err
+			}
+			// This is some submount, we can ignore this error for now, the final unmount will fail if this is a real problem
+			log.G(ctx).WithError(err).Debugf("failed to unmount submount %s", m.Mountpoint)
+		}
+	}
+	return nil
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+//
+// Because of the way `os.Remove` (and by extension `os.RemoveAll`) works, there
+// can be a race between reading directory entries and then actually attempting
+// to remove everything in the directory.
+// These types of errors do not need to be returned since it's ok for the dir to
+// be gone we can just retry the remove operation.
+//
+// This should not return a `os.ErrNotExist` kind of error under any circumstances
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	notExistErr := make(map[string]bool)
+
+	// track retries
+	exitOnErr := make(map[string]int)
+	maxRetry := 50
+
+	// Attempt to unmount anything beneath this dir first.
+	if err := unmountRecursive(ctx, dir); err != nil {
+		log.G(ctx).WithError(err).Debugf("failed to do initial unmount of %s", dir)
+	}
+
+	for {
+		err := os.RemoveAll(dir)
+		if err == nil {
+			return nil
+		}
+
+		pe, ok := err.(*os.PathError)
+		if !ok {
+			return err
+		}
+
+		if os.IsNotExist(err) {
+			if notExistErr[pe.Path] {
+				return err
+			}
+			notExistErr[pe.Path] = true
+
+			// There is a race where some subdir can be removed but after the
+			// parent dir entries have been read.
+			// So the path could be from `os.Remove(subdir)`
+			// If the reported non-existent path is not the passed in `dir` we
+			// should just retry, but otherwise return with no error.
+			if pe.Path == dir {
+				return nil
+			}
+			continue
+		}
+
+		if pe.Err != syscall.EBUSY {
+			return err
+		}
+		if e := mount.Unmount(pe.Path, unix.MNT_DETACH); e != nil {
+			return fmt.Errorf("error while removing %s: %w", dir, e)
+		}
+
+		if exitOnErr[pe.Path] == maxRetry {
+			return err
+		}
+		exitOnErr[pe.Path]++
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+var vmbasedRuntimes = []string{
+	"io.containerd.kata",
+}
+
+func isVMBasedRuntime(runtimeType string) bool {
+	for _, rt := range vmbasedRuntimes {
+		if strings.Contains(runtimeType, rt) {
+			return true
+		}
+	}
+	return false
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	if !isVMBasedRuntime(runtimeType) {
+		return nil
+	}
+	l, err := seutil.ChangeToKVM(spec.Process.SelinuxLabel)
+	if err != nil {
+		return fmt.Errorf("failed to get selinux kvm label: %w", err)
+	}
+	spec.Process.SelinuxLabel = l
+	return nil
+}

pkg/cri/sbserver/podsandbox/helpers_linux_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package sbserver
+package podsandbox
 
 import (
 	"context"

pkg/cri/sbserver/podsandbox/helpers_other.go

@@ -0,0 +1,39 @@
+//go:build !windows && !linux
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/cri/sbserver/podsandbox/helpers_selinux_linux_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package sbserver
+package podsandbox
 
 import (
 	"testing"

pkg/cri/sbserver/podsandbox/helpers_test.go

@@ -0,0 +1,483 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/protobuf/types"
+	"github.com/containerd/containerd/reference/docker"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/typeurl"
+
+	imagedigest "github.com/opencontainers/go-digest"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestGetUserFromImage tests the logic of getting image uid or user name of image user.
+func TestGetUserFromImage(t *testing.T) {
+	newI64 := func(i int64) *int64 { return &i }
+	for c, test := range map[string]struct {
+		user string
+		uid  *int64
+		name string
+	}{
+		"no gid": {
+			user: "0",
+			uid:  newI64(0),
+		},
+		"uid/gid": {
+			user: "0:1",
+			uid:  newI64(0),
+		},
+		"empty user": {
+			user: "",
+		},
+		"multiple separators": {
+			user: "1:2:3",
+			uid:  newI64(1),
+		},
+		"root username": {
+			user: "root:root",
+			name: "root",
+		},
+		"username": {
+			user: "test:test",
+			name: "test",
+		},
+	} {
+		t.Run(c, func(t *testing.T) {
+			actualUID, actualName := getUserFromImage(test.user)
+			assert.Equal(t, test.uid, actualUID)
+			assert.Equal(t, test.name, actualName)
+		})
+	}
+}
+
+func TestGetRepoDigestAndTag(t *testing.T) {
+	digest := imagedigest.Digest("sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582")
+	for desc, test := range map[string]struct {
+		ref                string
+		schema1            bool
+		expectedRepoDigest string
+		expectedRepoTag    string
+	}{
+		"repo tag should be empty if original ref has no tag": {
+			ref:                "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+		},
+		"repo tag should not be empty if original ref has tag": {
+			ref:                "gcr.io/library/busybox:latest",
+			expectedRepoDigest: "gcr.io/library/busybox@" + digest.String(),
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should be empty if original ref is schema1 and has no digest": {
+			ref:                "gcr.io/library/busybox:latest",
+			schema1:            true,
+			expectedRepoDigest: "",
+			expectedRepoTag:    "gcr.io/library/busybox:latest",
+		},
+		"repo digest should not be empty if original ref is schema1 but has digest": {
+			ref:                "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			schema1:            true,
+			expectedRepoDigest: "gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59594",
+			expectedRepoTag:    "",
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			named, err := docker.ParseDockerRef(test.ref)
+			assert.NoError(t, err)
+			repoDigest, repoTag := getRepoDigestAndTag(named, digest, test.schema1)
+			assert.Equal(t, test.expectedRepoDigest, repoDigest)
+			assert.Equal(t, test.expectedRepoTag, repoTag)
+		})
+	}
+}
+
+func TestBuildLabels(t *testing.T) {
+	imageConfigLabels := map[string]string{
+		"a":          "z",
+		"d":          "y",
+		"long-label": strings.Repeat("example", 10000),
+	}
+	configLabels := map[string]string{
+		"a": "b",
+		"c": "d",
+	}
+	newLabels := buildLabels(configLabels, imageConfigLabels, containerKindSandbox)
+	assert.Len(t, newLabels, 4)
+	assert.Equal(t, "b", newLabels["a"])
+	assert.Equal(t, "d", newLabels["c"])
+	assert.Equal(t, "y", newLabels["d"])
+	assert.Equal(t, containerKindSandbox, newLabels[containerKindLabel])
+	assert.NotContains(t, newLabels, "long-label")
+
+	newLabels["a"] = "e"
+	assert.Empty(t, configLabels[containerKindLabel], "should not add new labels into original label")
+	assert.Equal(t, "b", configLabels["a"], "change in new labels should not affect original label")
+}
+
+func TestParseImageReferences(t *testing.T) {
+	refs := []string{
+		"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"gcr.io/library/busybox:1.2",
+		"sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582",
+		"arbitrary-ref",
+	}
+	expectedTags := []string{
+		"gcr.io/library/busybox:1.2",
+	}
+	expectedDigests := []string{"gcr.io/library/busybox@sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582"}
+	tags, digests := parseImageReferences(refs)
+	assert.Equal(t, expectedTags, tags)
+	assert.Equal(t, expectedDigests, digests)
+}
+
+func TestGenerateRuntimeOptions(t *testing.T) {
+	nilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+`
+	nonNilOpts := `
+systemd_cgroup = true
+[containerd]
+  no_pivot = true
+  default_runtime_name = "default"
+[containerd.runtimes.legacy]
+  runtime_type = "` + plugin.RuntimeLinuxV1 + `"
+[containerd.runtimes.legacy.options]
+  Runtime = "legacy"
+  RuntimeRoot = "/legacy"
+[containerd.runtimes.runc]
+  runtime_type = "` + plugin.RuntimeRuncV1 + `"
+[containerd.runtimes.runc.options]
+  BinaryName = "runc"
+  Root = "/runc"
+  NoNewKeyring = true
+[containerd.runtimes.runcv2]
+  runtime_type = "` + plugin.RuntimeRuncV2 + `"
+[containerd.runtimes.runcv2.options]
+  BinaryName = "runc"
+  Root = "/runcv2"
+  NoNewKeyring = true
+`
+	var nilOptsConfig, nonNilOptsConfig criconfig.Config
+	tree, err := toml.Load(nilOpts)
+	require.NoError(t, err)
+	err = tree.Unmarshal(&nilOptsConfig)
+	require.NoError(t, err)
+	require.Len(t, nilOptsConfig.Runtimes, 3)
+
+	tree, err = toml.Load(nonNilOpts)
+	require.NoError(t, err)
+	err = tree.Unmarshal(&nonNilOptsConfig)
+	require.NoError(t, err)
+	require.Len(t, nonNilOptsConfig.Runtimes, 3)
+
+	for desc, test := range map[string]struct {
+		r               criconfig.Runtime
+		c               criconfig.Config
+		expectedOptions interface{}
+	}{
+		"when options is nil, should return nil option for io.containerd.runc.v1": {
+			r:               nilOptsConfig.Runtimes["runc"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should return nil option for io.containerd.runc.v2": {
+			r:               nilOptsConfig.Runtimes["runcv2"],
+			c:               nilOptsConfig,
+			expectedOptions: nil,
+		},
+		"when options is nil, should use legacy fields for legacy runtime": {
+			r: nilOptsConfig.Runtimes["legacy"],
+			c: nilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				SystemdCgroup: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v1": {
+			r: nonNilOptsConfig.Runtimes["runc"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runc",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for io.containerd.runc.v2": {
+			r: nonNilOptsConfig.Runtimes["runcv2"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runcoptions.Options{
+				BinaryName:   "runc",
+				Root:         "/runcv2",
+				NoNewKeyring: true,
+			},
+		},
+		"when options is not nil, should be able to decode for legacy runtime": {
+			r: nonNilOptsConfig.Runtimes["legacy"],
+			c: nonNilOptsConfig,
+			expectedOptions: &runctypes.RuncOptions{
+				Runtime:     "legacy",
+				RuntimeRoot: "/legacy",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			opts, err := generateRuntimeOptions(test.r, test.c)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedOptions, opts)
+		})
+	}
+}
+
+func TestEnvDeduplication(t *testing.T) {
+	for desc, test := range map[string]struct {
+		existing []string
+		kv       [][2]string
+		expected []string
+	}{
+		"single env": {
+			kv: [][2]string{
+				{"a", "b"},
+			},
+			expected: []string{"a=b"},
+		},
+		"multiple envs": {
+			kv: [][2]string{
+				{"a", "b"},
+				{"c", "d"},
+				{"e", "f"},
+			},
+			expected: []string{
+				"a=b",
+				"c=d",
+				"e=f",
+			},
+		},
+		"env override": {
+			kv: [][2]string{
+				{"k1", "v1"},
+				{"k2", "v2"},
+				{"k3", "v3"},
+				{"k3", "v4"},
+				{"k1", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v5",
+				"k2=v2",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+		"existing env": {
+			existing: []string{
+				"k1=v1",
+				"k2=v2",
+				"k3=v3",
+			},
+			kv: [][2]string{
+				{"k3", "v4"},
+				{"k2", "v5"},
+				{"k4", "v6"},
+			},
+			expected: []string{
+				"k1=v1",
+				"k2=v5",
+				"k3=v4",
+				"k4=v6",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			var spec runtimespec.Spec
+			if len(test.existing) > 0 {
+				spec.Process = &runtimespec.Process{
+					Env: test.existing,
+				}
+			}
+			for _, kv := range test.kv {
+				oci.WithEnv([]string{kv[0] + "=" + kv[1]})(context.Background(), nil, nil, &spec)
+			}
+			assert.Equal(t, test.expected, spec.Process.Env)
+		})
+	}
+}
+
+func TestPassThroughAnnotationsFilter(t *testing.T) {
+	for desc, test := range map[string]struct {
+		podAnnotations         map[string]string
+		runtimePodAnnotations  []string
+		passthroughAnnotations map[string]string
+	}{
+		"should support direct match": {
+			podAnnotations:         map[string]string{"c": "d", "d": "e"},
+			runtimePodAnnotations:  []string{"c"},
+			passthroughAnnotations: map[string]string{"c": "d"},
+		},
+		"should support wildcard match": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*.f", "z*g", "y.c*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"y.ca": "b",
+			},
+		},
+		"should support wildcard match all": {
+			podAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+			runtimePodAnnotations: []string{"*"},
+			passthroughAnnotations: map[string]string{
+				"t.f":  "j",
+				"z.g":  "o",
+				"z":    "o",
+				"y.ca": "b",
+				"y":    "b",
+			},
+		},
+		"should support match including path separator": {
+			podAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchend.com/1end":   "3",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchmid.com/mid1":   "6",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchhead.com/head1": "9",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"1end/matchend":       "14",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"mid1/matchmid":       "17",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"head1/matchhead":     "20",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+				"nomatch/nomatch":     "24",
+			},
+			runtimePodAnnotations: []string{
+				"matchend.com/end*",
+				"matchmid.com/mi*d",
+				"matchhead.com/*head",
+				"matchall.com/*",
+				"end*/matchend",
+				"mi*d/matchmid",
+				"*head/matchhead",
+				"*/matchall",
+				"match*/match*",
+			},
+			passthroughAnnotations: map[string]string{
+				"matchend.com/end":    "1",
+				"matchend.com/end1":   "2",
+				"matchmid.com/mid":    "4",
+				"matchmid.com/mi1d":   "5",
+				"matchhead.com/head":  "7",
+				"matchhead.com/1head": "8",
+				"matchall.com/abc":    "10",
+				"matchall.com/def":    "11",
+				"end/matchend":        "12",
+				"end1/matchend":       "13",
+				"mid/matchmid":        "15",
+				"mi1d/matchmid":       "16",
+				"head/matchhead":      "18",
+				"1head/matchhead":     "19",
+				"abc/matchall":        "21",
+				"def/matchall":        "22",
+				"match1/match2":       "23",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			passthroughAnnotations := getPassthroughAnnotations(test.podAnnotations, test.runtimePodAnnotations)
+			assert.Equal(t, test.passthroughAnnotations, passthroughAnnotations)
+		})
+	}
+}
+
+func TestEnsureRemoveAllNotExist(t *testing.T) {
+	// should never return an error for a non-existent path
+	if err := ensureRemoveAll(context.Background(), "/non/existent/path"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithDir(t *testing.T) {
+	dir := t.TempDir()
+	if err := ensureRemoveAll(context.Background(), dir); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEnsureRemoveAllWithFile(t *testing.T) {
+	tmp, err := os.CreateTemp("", "test-ensure-removeall-with-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmp.Close()
+	if err := ensureRemoveAll(context.Background(), tmp.Name()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetRuntimeOptions(t *testing.T) {
+	_, err := getRuntimeOptions(containers.Container{})
+	require.NoError(t, err)
+
+	var pbany *types.Any               // This is nil.
+	var typeurlAny typeurl.Any = pbany // This is typed nil.
+	_, err = getRuntimeOptions(containers.Container{Runtime: containers.RuntimeInfo{Options: typeurlAny}})
+	require.NoError(t, err)
+}

pkg/cri/sbserver/podsandbox/helpers_windows.go

@@ -0,0 +1,33 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"os"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// ensureRemoveAll is a wrapper for os.RemoveAll on Windows.
+func ensureRemoveAll(_ context.Context, dir string) error {
+	return os.RemoveAll(dir)
+}
+
+func modifyProcessLabel(runtimeType string, spec *specs.Spec) error {
+	return nil
+}

pkg/cri/sbserver/podsandbox/opts.go

@@ -0,0 +1,51 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+)
+
+// WithNRISandboxDelete calls delete for a sandbox'd task
+func WithNRISandboxDelete(sandboxID string) containerd.ProcessDeleteOpts {
+	return func(ctx context.Context, p containerd.Process) error {
+		task, ok := p.(containerd.Task)
+		if !ok {
+			return nil
+		}
+		nric, err := nri.New()
+		if err != nil {
+			log.G(ctx).WithError(err).Error("unable to create nri client")
+			return nil
+		}
+		if nric == nil {
+			return nil
+		}
+		sb := &nri.Sandbox{
+			ID: sandboxID,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Delete, sb); err != nil {
+			log.G(ctx).WithError(err).Errorf("Failed to delete nri for %q", task.ID())
+		}
+		return nil
+	}
+}

pkg/cri/sbserver/podsandbox/sandbox_run.go

@@ -0,0 +1,315 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/containerd/containerd"
+	api "github.com/containerd/containerd/api/services/sandbox/v1"
+	containerdio "github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/pkg/cri/annotations"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	customopts "github.com/containerd/containerd/pkg/cri/opts"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
+	"github.com/containerd/containerd/protobuf"
+	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/nri"
+	v1 "github.com/containerd/nri/types/v1"
+	"github.com/containerd/typeurl"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/opencontainers/selinux/go-selinux"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func init() {
+	typeurl.Register(&sandboxstore.Metadata{},
+		"github.com/containerd/cri/pkg/store/sandbox", "Metadata")
+}
+
+func (c *Controller) Start(ctx context.Context, id string) (_ *api.ControllerStartResponse, retErr error) {
+	sandboxInfo, err := c.client.SandboxStore().Get(ctx, id)
+	if err != nil {
+		return nil, fmt.Errorf("unable to find sandbox with id %q: %w", id, err)
+	}
+
+	var metadata sandboxstore.Metadata
+	if err := sandboxInfo.GetExtension("metadata", &metadata); err != nil {
+		return nil, fmt.Errorf("failed to get sandbox %q metadata: %w", id, err)
+	}
+
+	var (
+		config = metadata.Config
+		labels = map[string]string{}
+	)
+
+	// Ensure sandbox container image snapshot.
+	image, err := c.cri.EnsureImageExists(ctx, c.config.SandboxImage, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox image %q: %w", c.config.SandboxImage, err)
+	}
+
+	containerdImage, err := c.toContainerdImage(ctx, *image)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
+	}
+
+	ociRuntime, err := c.getSandboxRuntime(config, sandboxInfo.Runtime.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
+	}
+	log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)
+
+	labels["oci_runtime_type"] = ociRuntime.Type
+
+	// Create sandbox container.
+	// NOTE: sandboxContainerSpec SHOULD NOT have side
+	// effect, e.g. accessing/creating files, so that we can test
+	// it safely.
+	spec, err := c.sandboxContainerSpec(id, config, &image.ImageSpec.Config, metadata.NetNSPath, ociRuntime.PodAnnotations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate sandbox container spec: %w", err)
+	}
+	log.G(ctx).WithField("podsandboxid", id).Debugf("sandbox container spec: %#+v", spew.NewFormatter(spec))
+
+	metadata.ProcessLabel = spec.Process.SelinuxLabel
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(metadata.ProcessLabel)
+		}
+	}()
+	labels["selinux_label"] = metadata.ProcessLabel
+
+	// handle any KVM based runtime
+	if err := modifyProcessLabel(ociRuntime.Type, spec); err != nil {
+		return nil, err
+	}
+
+	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+		// If privileged don't set selinux label, but we still record the MCS label so that
+		// the unused label can be freed later.
+		spec.Process.SelinuxLabel = ""
+	}
+
+	// Generate spec options that will be applied to the spec later.
+	specOpts, err := c.sandboxContainerSpecOpts(config, &image.ImageSpec.Config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate sandbox container spec options: %w", err)
+	}
+
+	sandboxLabels := buildLabels(config.Labels, image.ImageSpec.Config.Labels, containerKindSandbox)
+
+	runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate runtime options: %w", err)
+	}
+
+	snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
+	opts := []containerd.NewContainerOpts{
+		containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)),
+		customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt),
+		containerd.WithSpec(spec, specOpts...),
+		containerd.WithContainerLabels(sandboxLabels),
+		containerd.WithContainerExtension(sandboxMetadataExtension, &metadata),
+		containerd.WithRuntime(ociRuntime.Type, runtimeOpts)}
+
+	container, err := c.client.NewContainer(ctx, id, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create containerd container: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			if err := container.Delete(deferCtx, containerd.WithSnapshotCleanup); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to delete containerd container %q", id)
+			}
+		}
+	}()
+
+	// Create sandbox container root directories.
+	sandboxRootDir := c.getSandboxRootDir(id)
+	if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create sandbox root directory %q: %w",
+			sandboxRootDir, err)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the sandbox root directory.
+			if err := c.os.RemoveAll(sandboxRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove sandbox root directory %q",
+					sandboxRootDir)
+			}
+		}
+	}()
+
+	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
+	if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create volatile sandbox root directory %q: %w",
+			volatileSandboxRootDir, err)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile sandbox root directory.
+			if err := c.os.RemoveAll(volatileSandboxRootDir); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to remove volatile sandbox root directory %q",
+					volatileSandboxRootDir)
+			}
+		}
+	}()
+
+	// Setup files required for the sandbox.
+	if err = c.setupSandboxFiles(id, config); err != nil {
+		return nil, fmt.Errorf("failed to setup sandbox files: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			if err = c.cleanupSandboxFiles(id, config); err != nil {
+				log.G(ctx).WithError(err).Errorf("Failed to cleanup sandbox files in %q",
+					sandboxRootDir)
+			}
+		}
+	}()
+
+	// Update sandbox created timestamp.
+	info, err := container.Info(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox container info: %w", err)
+	}
+
+	// Create sandbox task in containerd.
+	log.G(ctx).Tracef("Create sandbox container (id=%q, name=%q).", id, metadata.Name)
+
+	taskOpts := c.taskOpts(ociRuntime.Type)
+	if ociRuntime.Path != "" {
+		taskOpts = append(taskOpts, containerd.WithRuntimePath(ociRuntime.Path))
+	}
+
+	// We don't need stdio for sandbox container.
+	task, err := container.NewTask(ctx, containerdio.NullIO, taskOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create containerd task: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			deferCtx, deferCancel := ctrdutil.DeferContext()
+			defer deferCancel()
+			// Cleanup the sandbox container if an error is returned.
+			if _, err := task.Delete(deferCtx, WithNRISandboxDelete(id), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
+				log.G(ctx).WithError(err).Errorf("Failed to delete sandbox container %q", id)
+			}
+		}
+	}()
+
+	// wait is a long running background request, no timeout needed.
+	exitCh, err := task.Wait(ctrdutil.NamespacedContext())
+	if err != nil {
+		return nil, fmt.Errorf("failed to wait for sandbox container task: %w", err)
+	}
+
+	nric, err := nri.New()
+	if err != nil {
+		return nil, fmt.Errorf("unable to create nri client: %w", err)
+	}
+	if nric != nil {
+		nriSB := &nri.Sandbox{
+			ID:     id,
+			Labels: config.Labels,
+		}
+		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil {
+			return nil, fmt.Errorf("nri invoke: %w", err)
+		}
+	}
+
+	if err := task.Start(ctx); err != nil {
+		return nil, fmt.Errorf("failed to start sandbox container task %q: %w", id, err)
+	}
+
+	// start the monitor after adding sandbox into the store, this ensures
+	// that sandbox is in the store, when event monitor receives the TaskExit event.
+	//
+	// TaskOOM from containerd may come before sandbox is added to store,
+	// but we don't care about sandbox TaskOOM right now, so it is fine.
+	c.cri.StartSandboxExitMonitor(context.Background(), id, task.Pid(), exitCh) // TODO: Move back to CRI service.
+
+	resp := &api.ControllerStartResponse{
+		SandboxID: id,
+		Pid:       task.Pid(),
+		CreatedAt: protobuf.ToTimestamp(info.CreatedAt),
+	}
+
+	return resp, nil
+}
+
+// untrustedWorkload returns true if the sandbox contains untrusted workload.
+func untrustedWorkload(config *runtime.PodSandboxConfig) bool {
+	return config.GetAnnotations()[annotations.UntrustedWorkload] == "true"
+}
+
+// hostAccessingSandbox returns true if the sandbox configuration
+// requires additional host access for the sandbox.
+func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
+	securityContext := config.GetLinux().GetSecurityContext()
+
+	namespaceOptions := securityContext.GetNamespaceOptions()
+	if namespaceOptions.GetNetwork() == runtime.NamespaceMode_NODE ||
+		namespaceOptions.GetPid() == runtime.NamespaceMode_NODE ||
+		namespaceOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		return true
+	}
+
+	return false
+}
+
+// getSandboxRuntime returns the runtime configuration for sandbox.
+// If the sandbox contains untrusted workload, runtime for untrusted workload will be returned,
+// or else default runtime will be returned.
+func (c *Controller) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
+	if untrustedWorkload(config) {
+		// If the untrusted annotation is provided, runtimeHandler MUST be empty.
+		if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
+			return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
+		}
+
+		//  If the untrusted workload is requesting access to the host/node, this request will fail.
+		//
+		//  Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
+		// runtime may support this.  For example, in a virtual-machine isolated runtime, privileged
+		// is a supported option, granting the workload to access the entire guest VM instead of host.
+		// TODO(windows): Deprecate this so that we don't need to handle it for windows.
+		if hostAccessingSandbox(config) {
+			return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
+		}
+
+		runtimeHandler = criconfig.RuntimeUntrusted
+	}
+
+	if runtimeHandler == "" {
+		runtimeHandler = c.config.ContainerdConfig.DefaultRuntimeName
+	}
+
+	handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
+	if !ok {
+		return criconfig.Runtime{}, fmt.Errorf("no runtime for %q is configured", runtimeHandler)
+	}
+	return handler, nil
+}

pkg/cri/sbserver/podsandbox/sandbox_run_linux.go

@@ -0,0 +1,350 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/plugin"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux"
+	"golang.org/x/sys/unix"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/containerd/containerd/pkg/cri/annotations"
+	customopts "github.com/containerd/containerd/pkg/cri/opts"
+	osinterface "github.com/containerd/containerd/pkg/os"
+	"github.com/containerd/containerd/pkg/userns"
+)
+
+func (c *Controller) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
+	// Creates a spec Generator with the default spec.
+	// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
+	specOpts := []oci.SpecOpts{
+		oci.WithoutRunMount,
+		customopts.WithoutDefaultSecuritySettings,
+		customopts.WithRelativeRoot(relativeRootfsPath),
+		oci.WithEnv(imageConfig.Env),
+		oci.WithRootFSReadonly(),
+		oci.WithHostname(config.GetHostname()),
+	}
+	if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
+		// Pause image must have entrypoint or cmd.
+		return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
+	}
+	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
+
+	// Set cgroups parent.
+	if c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDisabledCgroups)
+	} else {
+		if config.GetLinux().GetCgroupParent() != "" {
+			cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id)
+			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
+		}
+	}
+
+	// When cgroup parent is not set, containerd-shim will create container in a child cgroup
+	// of the cgroup itself is in.
+	// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
+
+	// Set namespace options.
+	var (
+		securityContext = config.GetLinux().GetSecurityContext()
+		nsOptions       = securityContext.GetNamespaceOptions()
+	)
+	if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace))
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.UTSNamespace))
+	} else {
+		specOpts = append(specOpts, oci.WithLinuxNamespace(
+			runtimespec.LinuxNamespace{
+				Type: runtimespec.NetworkNamespace,
+				Path: nsPath,
+			}))
+	}
+	if nsOptions.GetPid() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace))
+	}
+	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace))
+	}
+
+	// It's fine to generate the spec before the sandbox /dev/shm
+	// is actually created.
+	sandboxDevShm := c.getSandboxDevShm(id)
+	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
+		sandboxDevShm = devShm
+	}
+	// Remove the default /dev/shm mount from defaultMounts, it is added in oci/mounts.go.
+	specOpts = append(specOpts, oci.WithoutMounts(devShm))
+	// In future the when user-namespace is enabled, the `nosuid, nodev, noexec` flags are
+	// required, otherwise the remount will fail with EPERM. Just use them unconditionally,
+	// they are nice to have anyways.
+	specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{
+		{
+			Source:      sandboxDevShm,
+			Destination: devShm,
+			Type:        "bind",
+			Options:     []string{"rbind", "ro", "nosuid", "nodev", "noexec"},
+		},
+		// Add resolv.conf for katacontainers to setup the DNS of pod VM properly.
+		{
+			Source:      c.getResolvPath(id),
+			Destination: resolvConfPath,
+			Type:        "bind",
+			Options:     []string{"rbind", "ro"},
+		},
+	}))
+
+	processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions())
+	if err != nil {
+		return nil, fmt.Errorf("failed to init selinux options %+v: %w", securityContext.GetSelinuxOptions(), err)
+	}
+	defer func() {
+		if retErr != nil {
+			selinux.ReleaseLabel(processLabel)
+		}
+	}()
+
+	supplementalGroups := securityContext.GetSupplementalGroups()
+	specOpts = append(specOpts,
+		customopts.WithSelinuxLabels(processLabel, mountLabel),
+		customopts.WithSupplementalGroups(supplementalGroups),
+	)
+
+	// Add sysctls
+	sysctls := config.GetLinux().GetSysctls()
+	if sysctls == nil {
+		sysctls = make(map[string]string)
+	}
+	_, ipUnprivilegedPortStart := sysctls["net.ipv4.ip_unprivileged_port_start"]
+	_, pingGroupRange := sysctls["net.ipv4.ping_group_range"]
+	if nsOptions.GetNetwork() != runtime.NamespaceMode_NODE {
+		if c.config.EnableUnprivilegedPorts && !ipUnprivilegedPortStart {
+			sysctls["net.ipv4.ip_unprivileged_port_start"] = "0"
+		}
+		if c.config.EnableUnprivilegedICMP && !pingGroupRange && !userns.RunningInUserNS() {
+			sysctls["net.ipv4.ping_group_range"] = "0 2147483647"
+		}
+	}
+	specOpts = append(specOpts, customopts.WithSysctls(sysctls))
+
+	// Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile
+
+	if !c.config.DisableCgroup {
+		specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
+	}
+
+	if res := config.GetLinux().GetResources(); res != nil {
+		specOpts = append(specOpts,
+			customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)),
+			customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)),
+			customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)),
+			customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10)))
+	}
+
+	specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		runtimePodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
+		customopts.WithAnnotation(annotations.SandboxID, id),
+		customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
+		customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
+		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
+	)
+
+	return c.runtimeSpec(id, "", specOpts...)
+}
+
+// sandboxContainerSpecOpts generates OCI spec options for
+// the sandbox container.
+func (c *Controller) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	var (
+		securityContext = config.GetLinux().GetSecurityContext()
+		specOpts        []oci.SpecOpts
+		err             error
+	)
+	ssp := securityContext.GetSeccomp()
+	if ssp == nil {
+		ssp, err = generateSeccompSecurityProfile(
+			securityContext.GetSeccompProfilePath(), //nolint:staticcheck // Deprecated but we don't want to remove yet
+			c.config.UnsetSeccompProfile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
+		}
+	}
+	seccompSpecOpts, err := c.generateSeccompSpecOpts(
+		ssp,
+		securityContext.GetPrivileged(),
+		c.seccompEnabled())
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
+	}
+	if seccompSpecOpts != nil {
+		specOpts = append(specOpts, seccompSpecOpts)
+	}
+
+	userstr, err := generateUserString(
+		"",
+		securityContext.GetRunAsUser(),
+		securityContext.GetRunAsGroup(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate user string: %w", err)
+	}
+	if userstr == "" {
+		// Lastly, since no user override was passed via CRI try to set via OCI
+		// Image
+		userstr = imageConfig.User
+	}
+	if userstr != "" {
+		specOpts = append(specOpts, oci.WithUser(userstr))
+	}
+	return specOpts, nil
+}
+
+// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
+// /etc/resolv.conf and /etc/hostname.
+func (c *Controller) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	sandboxEtcHostname := c.getSandboxHostname(id)
+	hostname := config.GetHostname()
+	if hostname == "" {
+		var err error
+		hostname, err = c.os.Hostname()
+		if err != nil {
+			return fmt.Errorf("failed to get hostname: %w", err)
+		}
+	}
+	if err := c.os.WriteFile(sandboxEtcHostname, []byte(hostname+"\n"), 0644); err != nil {
+		return fmt.Errorf("failed to write hostname to %q: %w", sandboxEtcHostname, err)
+	}
+
+	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
+	sandboxEtcHosts := c.getSandboxHosts(id)
+	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
+		return fmt.Errorf("failed to generate sandbox hosts file %q: %w", sandboxEtcHosts, err)
+	}
+
+	// Set DNS options. Maintain a resolv.conf for the sandbox.
+	var err error
+	resolvContent := ""
+	if dnsConfig := config.GetDnsConfig(); dnsConfig != nil {
+		resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options)
+		if err != nil {
+			return fmt.Errorf("failed to parse sandbox DNSConfig %+v: %w", dnsConfig, err)
+		}
+	}
+	resolvPath := c.getResolvPath(id)
+	if resolvContent == "" {
+		// copy host's resolv.conf to resolvPath
+		err = c.os.CopyFile(resolvConfPath, resolvPath, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to copy host's resolv.conf to %q: %w", resolvPath, err)
+		}
+	} else {
+		err = c.os.WriteFile(resolvPath, []byte(resolvContent), 0644)
+		if err != nil {
+			return fmt.Errorf("failed to write resolv content to %q: %w", resolvPath, err)
+		}
+	}
+
+	// Setup sandbox /dev/shm.
+	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
+		if _, err := c.os.Stat(devShm); err != nil {
+			return fmt.Errorf("host %q is not available for host ipc: %w", devShm, err)
+		}
+	} else {
+		sandboxDevShm := c.getSandboxDevShm(id)
+		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
+			return fmt.Errorf("failed to create sandbox shm: %w", err)
+		}
+		shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
+		if err := c.os.(osinterface.UNIX).Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
+			return fmt.Errorf("failed to mount sandbox shm: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// parseDNSOptions parse DNS options into resolv.conf format content,
+// if none option is specified, will return empty with no error.
+func parseDNSOptions(servers, searches, options []string) (string, error) {
+	resolvContent := ""
+
+	if len(searches) > 0 {
+		resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " "))
+	}
+
+	if len(servers) > 0 {
+		resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver "))
+	}
+
+	if len(options) > 0 {
+		resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " "))
+	}
+
+	return resolvContent, nil
+}
+
+// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
+// remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
+func (c *Controller) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() != runtime.NamespaceMode_NODE {
+		path, err := c.os.FollowSymlinkInScope(c.getSandboxDevShm(id), "/")
+		if err != nil {
+			return fmt.Errorf("failed to follow symlink: %w", err)
+		}
+		if err := c.os.(osinterface.UNIX).Unmount(path); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("failed to unmount %q: %w", path, err)
+		}
+	}
+	return nil
+}
+
+// taskOpts generates task options for a (sandbox) container.
+func (c *Controller) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	// TODO(random-liu): Remove this after shim v1 is deprecated.
+	var taskOpts []containerd.NewTaskOpts
+
+	// c.config.NoPivot is only supported for RuntimeLinuxV1 = "io.containerd.runtime.v1.linux" legacy linux runtime
+	// and is not supported for RuntimeRuncV1 = "io.containerd.runc.v1" or  RuntimeRuncV2 = "io.containerd.runc.v2"
+	// for RuncV1/2 no pivot is set under the containerd.runtimes.runc.options config see
+	// https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26
+	if c.config.NoPivot && runtimeType == plugin.RuntimeLinuxV1 {
+		taskOpts = append(taskOpts, containerd.WithNoPivotRoot)
+	}
+
+	return taskOpts
+}

pkg/cri/sbserver/podsandbox/sandbox_run_linux_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package sbserver
+package podsandbox
 
 import (
 	"os"
@@ -236,7 +236,7 @@ func TestLinuxSandboxContainerSpec(t *testing.T) {
 		},
 	} {
 		t.Run(desc, func(t *testing.T) {
-			c := newTestCRIService()
+			c := newControllerService()
 			c.config.EnableUnprivilegedICMP = true
 			c.config.EnableUnprivilegedPorts = true
 			config, imageConfig, specCheck := getRunPodSandboxTestData()
@@ -428,7 +428,7 @@ options timeout:1
 		},
 	} {
 		t.Run(desc, func(t *testing.T) {
-			c := newTestCRIService()
+			c := newControllerService()
 			c.os.(*ostesting.FakeOS).HostnameFn = func() (string, error) {
 				return realhostname, nil
 			}
@@ -509,7 +509,7 @@ options timeout:1
 
 func TestSandboxDisableCgroup(t *testing.T) {
 	config, imageConfig, _ := getRunPodSandboxTestData()
-	c := newTestCRIService()
+	c := newControllerService()
 	c.config.DisableCgroup = true
 	spec, err := c.sandboxContainerSpec("test-id", config, imageConfig, "test-cni", []string{})
 	require.NoError(t, err)

pkg/cri/sbserver/podsandbox/sandbox_run_other.go

@@ -0,0 +1,56 @@
+//go:build !windows && !linux
+// +build !windows,!linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func (c *Controller) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
+	return c.runtimeSpec(id, "")
+}
+
+// sandboxContainerSpecOpts generates OCI spec options for
+// the sandbox container.
+func (c *Controller) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return []oci.SpecOpts{}, nil
+}
+
+// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
+// /etc/resolv.conf and /etc/hostname.
+func (c *Controller) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
+// remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
+func (c *Controller) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// taskOpts generates task options for a (sandbox) container.
+func (c *Controller) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	return []containerd.NewTaskOpts{}
+}

pkg/cri/sbserver/podsandbox/sandbox_run_other_test.go

@@ -17,7 +17,7 @@
    limitations under the License.
 */
 
-package sbserver
+package podsandbox
 
 import (
 	"testing"

pkg/cri/sbserver/podsandbox/sandbox_run_test.go

@@ -0,0 +1,372 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	goruntime "runtime"
+	"testing"
+
+	"github.com/containerd/typeurl"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/containerd/containerd/pkg/cri/annotations"
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
+)
+
+func TestSandboxContainerSpec(t *testing.T) {
+	switch goruntime.GOOS {
+	case "darwin":
+		t.Skip("not implemented on Darwin")
+	case "freebsd":
+		t.Skip("not implemented on FreeBSD")
+	}
+	testID := "test-id"
+	nsPath := "test-cni"
+	for desc, test := range map[string]struct {
+		configChange      func(*runtime.PodSandboxConfig)
+		podAnnotations    []string
+		imageConfigChange func(*imagespec.ImageConfig)
+		specCheck         func(*testing.T, *runtimespec.Spec)
+		expectErr         bool
+	}{
+		"should return error when entrypoint and cmd are empty": {
+			imageConfigChange: func(c *imagespec.ImageConfig) {
+				c.Entrypoint = nil
+				c.Cmd = nil
+			},
+			expectErr: true,
+		},
+		"a passthrough annotation should be passed as an OCI annotation": {
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+			},
+		},
+		"a non-passthrough annotation should not be passed as an OCI annotation": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["d"] = "e"
+			},
+			podAnnotations: []string{"c"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["c"], "d")
+				_, ok := spec.Annotations["d"]
+				assert.False(t, ok)
+			},
+		},
+		"passthrough annotations should support wildcard match": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Annotations["t.f"] = "j"
+				c.Annotations["z.g"] = "o"
+				c.Annotations["z"] = "o"
+				c.Annotations["y.ca"] = "b"
+				c.Annotations["y"] = "b"
+			},
+			podAnnotations: []string{"t*", "z.*", "y.c*"},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				assert.Equal(t, spec.Annotations["t.f"], "j")
+				assert.Equal(t, spec.Annotations["z.g"], "o")
+				assert.Equal(t, spec.Annotations["y.ca"], "b")
+				_, ok := spec.Annotations["y"]
+				assert.False(t, ok)
+				_, ok = spec.Annotations["z"]
+				assert.False(t, ok)
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			c := newControllerService()
+			config, imageConfig, specCheck := getRunPodSandboxTestData()
+			if test.configChange != nil {
+				test.configChange(config)
+			}
+
+			if test.imageConfigChange != nil {
+				test.imageConfigChange(imageConfig)
+			}
+			spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath,
+				test.podAnnotations)
+			if test.expectErr {
+				assert.Error(t, err)
+				assert.Nil(t, spec)
+				return
+			}
+			assert.NoError(t, err)
+			assert.NotNil(t, spec)
+			specCheck(t, testID, spec)
+			if test.specCheck != nil {
+				test.specCheck(t, spec)
+			}
+		})
+	}
+}
+
+func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
+	for desc, test := range map[string]struct {
+		configChange func(*runtime.PodSandboxConfig)
+	}{
+		"should marshal original config": {},
+		"should marshal Linux": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				if c.Linux == nil {
+					c.Linux = &runtime.LinuxPodSandboxConfig{}
+				}
+				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						Network: runtime.NamespaceMode_NODE,
+						Pid:     runtime.NamespaceMode_NODE,
+						Ipc:     runtime.NamespaceMode_NODE,
+					},
+					SupplementalGroups: []int64{1111, 2222},
+				}
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			meta := &sandboxstore.Metadata{
+				ID:        "1",
+				Name:      "sandbox_1",
+				NetNSPath: "/home/cloud",
+			}
+			meta.Config, _, _ = getRunPodSandboxTestData()
+			if test.configChange != nil {
+				test.configChange(meta.Config)
+			}
+
+			any, err := typeurl.MarshalAny(meta)
+			assert.NoError(t, err)
+			data, err := typeurl.UnmarshalAny(any)
+			assert.NoError(t, err)
+			assert.IsType(t, &sandboxstore.Metadata{}, data)
+			curMeta, ok := data.(*sandboxstore.Metadata)
+			assert.True(t, ok)
+			assert.Equal(t, meta, curMeta)
+		})
+	}
+}
+
+func TestHostAccessingSandbox(t *testing.T) {
+	privilegedContext := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: true,
+			},
+		},
+	}
+	nonPrivilegedContext := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: false,
+			},
+		},
+	}
+	hostNamespace := &runtime.PodSandboxConfig{
+		Linux: &runtime.LinuxPodSandboxConfig{
+			SecurityContext: &runtime.LinuxSandboxSecurityContext{
+				Privileged: false,
+				NamespaceOptions: &runtime.NamespaceOption{
+					Network: runtime.NamespaceMode_NODE,
+					Pid:     runtime.NamespaceMode_NODE,
+					Ipc:     runtime.NamespaceMode_NODE,
+				},
+			},
+		},
+	}
+	tests := []struct {
+		name   string
+		config *runtime.PodSandboxConfig
+		want   bool
+	}{
+		{"Security Context is nil", nil, false},
+		{"Security Context is privileged", privilegedContext, false},
+		{"Security Context is not privileged", nonPrivilegedContext, false},
+		{"Security Context namespace host access", hostNamespace, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := hostAccessingSandbox(tt.config); got != tt.want {
+				t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetSandboxRuntime(t *testing.T) {
+	untrustedWorkloadRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "untrusted-workload-runtime",
+		Root:   "",
+	}
+
+	defaultRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "default-runtime",
+		Root:   "",
+	}
+
+	fooRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "foo-bar",
+		Root:   "",
+	}
+
+	for desc, test := range map[string]struct {
+		sandboxConfig   *runtime.PodSandboxConfig
+		runtimeHandler  string
+		runtimes        map[string]criconfig.Runtime
+		expectErr       bool
+		expectedRuntime criconfig.Runtime
+	}{
+		"should return error if untrusted workload requires host access": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Linux: &runtime.LinuxPodSandboxConfig{
+					SecurityContext: &runtime.LinuxSandboxSecurityContext{
+						Privileged: false,
+						NamespaceOptions: &runtime.NamespaceOption{
+							Network: runtime.NamespaceMode_NODE,
+							Pid:     runtime.NamespaceMode_NODE,
+							Ipc:     runtime.NamespaceMode_NODE,
+						},
+					},
+				},
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectErr: true,
+		},
+		"should use untrusted workload runtime for untrusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should use default runtime for regular workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
+			expectedRuntime: defaultRuntime,
+		},
+		"should use default runtime for trusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "false",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: defaultRuntime,
+		},
+		"should return error if untrusted workload runtime is required but not configured": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+			},
+			expectErr: true,
+		},
+		"should use 'untrusted' runtime for untrusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should use 'untrusted' runtime for untrusted workload & handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler: "untrusted",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+			},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should return an error if untrusted annotation with conflicting handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler: "foo",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
+			expectErr: true,
+		},
+		"should use correct runtime for a runtime handler": {
+			sandboxConfig:  &runtime.PodSandboxConfig{},
+			runtimeHandler: "foo",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault:   defaultRuntime,
+				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
+				"foo":                      fooRuntime,
+			},
+			expectedRuntime: fooRuntime,
+		},
+		"should return error if runtime handler is required but not configured": {
+			sandboxConfig:  &runtime.PodSandboxConfig{},
+			runtimeHandler: "bar",
+			runtimes: map[string]criconfig.Runtime{
+				criconfig.RuntimeDefault: defaultRuntime,
+				"foo":                    fooRuntime,
+			},
+			expectErr: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			cri := newControllerService()
+			cri.config = criconfig.Config{
+				PluginConfig: criconfig.DefaultConfig(),
+			}
+			cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
+			cri.config.ContainerdConfig.Runtimes = test.runtimes
+			r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
+			assert.Equal(t, test.expectErr, err != nil)
+			assert.Equal(t, test.expectedRuntime, r)
+		})
+	}
+}

pkg/cri/sbserver/podsandbox/sandbox_run_windows.go

@@ -0,0 +1,113 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package podsandbox
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/oci"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"github.com/containerd/containerd/pkg/cri/annotations"
+	customopts "github.com/containerd/containerd/pkg/cri/opts"
+)
+
+func (c *Controller) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
+	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
+	// Creates a spec Generator with the default spec.
+	specOpts := []oci.SpecOpts{
+		oci.WithEnv(imageConfig.Env),
+		oci.WithHostname(config.GetHostname()),
+	}
+	if imageConfig.WorkingDir != "" {
+		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
+	}
+
+	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
+		// Pause image must have entrypoint or cmd.
+		return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
+	}
+	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
+
+	specOpts = append(specOpts,
+		// Clear the root location since hcsshim expects it.
+		// NOTE: readonly rootfs doesn't work on windows.
+		customopts.WithoutRoot,
+		customopts.WithWindowsNetworkNamespace(nsPath),
+	)
+
+	specOpts = append(specOpts, customopts.WithWindowsDefaultSandboxShares)
+
+	// Start with the image config user and override below if RunAsUsername is not "".
+	username := imageConfig.User
+
+	runAsUser := config.GetWindows().GetSecurityContext().GetRunAsUsername()
+	if runAsUser != "" {
+		username = runAsUser
+	}
+
+	cs := config.GetWindows().GetSecurityContext().GetCredentialSpec()
+	if cs != "" {
+		specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs))
+	}
+
+	// There really isn't a good Windows way to verify that the username is available in the
+	// image as early as here like there is for Linux. Later on in the stack hcsshim
+	// will handle the behavior of erroring out if the user isn't available in the image
+	// when trying to run the init process.
+	specOpts = append(specOpts, oci.WithUser(username))
+
+	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
+		runtimePodAnnotations) {
+		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
+	}
+
+	specOpts = append(specOpts,
+		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
+		customopts.WithAnnotation(annotations.SandboxID, id),
+		customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
+		customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
+		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
+		customopts.WithAnnotation(annotations.WindowsHostProcess, strconv.FormatBool(config.GetWindows().GetSecurityContext().GetHostProcess())),
+	)
+
+	return c.runtimeSpec(id, "", specOpts...)
+}
+
+// No sandbox container spec options for windows yet.
+func (c *Controller) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
+	return nil, nil
+}
+
+// No sandbox files needed for windows.
+func (c *Controller) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// No sandbox files needed for windows.
+func (c *Controller) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
+	return nil
+}
+
+// No task options needed for windows.
+func (c *Controller) taskOpts(runtimeType string) []containerd.NewTaskOpts {
+	return nil
+}

pkg/cri/sbserver/podsandbox/sandbox_run_windows_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package sbserver
+package podsandbox
 
 import (
 	"testing"
@@ -95,7 +95,7 @@ func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConf
 func TestSandboxWindowsNetworkNamespace(t *testing.T) {
 	testID := "test-id"
 	nsPath := "test-cni"
-	c := newTestCRIService()
+	c := newControllerService()
 
 	config, imageConfig, specCheck := getRunPodSandboxTestData()
 	spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil)

pkg/cri/sbserver/sandbox_remove.go

@@ -108,6 +108,10 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 	// 3) On-going operations which have held the reference will not be affected.
 	c.sandboxStore.Delete(id)
 
+	if err := c.client.SandboxStore().Delete(ctx, id); err != nil {
+		return nil, fmt.Errorf("failed to remove sandbox metadata from store: %w", err)
+	}
+
 	// Release the sandbox name reserved for the sandbox.
 	c.sandboxNameIndex.ReleaseByKey(id)
 

pkg/cri/sbserver/sandbox_run.go

@@ -27,28 +27,21 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containerd/containerd/protobuf"
+	sb "github.com/containerd/containerd/sandbox"
 	"github.com/containerd/go-cni"
-	"github.com/containerd/nri"
-	v1 "github.com/containerd/nri/types/v1"
 	"github.com/containerd/typeurl"
-	"github.com/davecgh/go-spew/spew"
-	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/sirupsen/logrus"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
-	"github.com/containerd/containerd"
-	containerdio "github.com/containerd/containerd/cio"
-	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/pkg/cri/annotations"
 	criconfig "github.com/containerd/containerd/pkg/cri/config"
-	customopts "github.com/containerd/containerd/pkg/cri/opts"
 	"github.com/containerd/containerd/pkg/cri/server/bandwidth"
 	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
 	"github.com/containerd/containerd/pkg/cri/util"
 	ctrdutil "github.com/containerd/containerd/pkg/cri/util"
 	"github.com/containerd/containerd/pkg/netns"
-	"github.com/containerd/containerd/snapshots"
 )
 
 func init() {
@@ -82,6 +75,15 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		}
 	}()
 
+	sandboxInfo := sb.Sandbox{
+		ID: id,
+		// TODO: runtime handler can be an empty string, should use default one and enable back validation of this field in metadata store.
+		Runtime: sb.RuntimeOpts{Name: r.GetRuntimeHandler()},
+	}
+
+	// Save sandbox name
+	sandboxInfo.AddLabel("name", name)
+
 	// Create initial internal sandbox object.
 	sandbox := sandboxstore.NewSandbox(
 		sandboxstore.Metadata{
@@ -95,23 +97,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		},
 	)
 
-	// Ensure sandbox container image snapshot.
+	var (
-	image, err := c.ensureImageExists(ctx, c.config.SandboxImage, config)
+		podNetwork = true
-	if err != nil {
+		err        error
-		return nil, fmt.Errorf("failed to get sandbox image %q: %w", c.config.SandboxImage, err)
+	)
-	}
-	containerdImage, err := c.toContainerdImage(ctx, *image)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get image from containerd %q: %w", image.ID, err)
-	}
-
-	ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler())
-	if err != nil {
-		return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
-	}
-	log.G(ctx).WithField("podsandboxid", id).Debugf("use OCI runtime %+v", ociRuntime)
-
-	podNetwork := true
 
 	if goruntime.GOOS != "windows" &&
 		config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtime.NamespaceMode_NODE {
@@ -169,176 +158,44 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		sandboxCreateNetworkTimer.UpdateSince(netStart)
 	}
 
+	// Save sandbox metadata to store
+	if err := sandboxInfo.AddExtension("metadata", &sandbox.Metadata); err != nil {
+		return nil, fmt.Errorf("unable to save sandbox %q to store: %w", id, err)
+	}
+
+	if _, err := c.client.SandboxStore().Create(ctx, sandboxInfo); err != nil {
+		return nil, fmt.Errorf("failed to save sandbox metadata: %w", err)
+	}
+
 	runtimeStart := time.Now()
-	// Create sandbox container.
+
-	// NOTE: sandboxContainerSpec SHOULD NOT have side
+	resp, err := c.sandboxController.Start(ctx, id)
-	// effect, e.g. accessing/creating files, so that we can test
-	// it safely.
-	spec, err := c.sandboxContainerSpec(id, config, &image.ImageSpec.Config, sandbox.NetNSPath, ociRuntime.PodAnnotations)
 	if err != nil {
-		return nil, fmt.Errorf("failed to generate sandbox container spec: %w", err)
+		return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err)
-	}
-	log.G(ctx).WithField("podsandboxid", id).Debugf("sandbox container spec: %#+v", spew.NewFormatter(spec))
-	sandbox.ProcessLabel = spec.Process.SelinuxLabel
-	defer func() {
-		if retErr != nil {
-			selinux.ReleaseLabel(sandbox.ProcessLabel)
-		}
-	}()
-
-	// handle any KVM based runtime
-	if err := modifyProcessLabel(ociRuntime.Type, spec); err != nil {
-		return nil, err
 	}
 
-	if config.GetLinux().GetSecurityContext().GetPrivileged() {
+	labels := resp.GetLabels()
-		// If privileged don't set selinux label, but we still record the MCS label so that
+	if labels == nil {
-		// the unused label can be freed later.
+		labels = map[string]string{}
-		spec.Process.SelinuxLabel = ""
 	}
 
-	// Generate spec options that will be applied to the spec later.
+	sandbox.ProcessLabel = labels["selinux_label"]
-	specOpts, err := c.sandboxContainerSpecOpts(config, &image.ImageSpec.Config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate sandbox container spec options: %w", err)
-	}
-
-	sandboxLabels := buildLabels(config.Labels, image.ImageSpec.Config.Labels, containerKindSandbox)
-
-	runtimeOpts, err := generateRuntimeOptions(ociRuntime, c.config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate runtime options: %w", err)
-	}
-	snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
-	opts := []containerd.NewContainerOpts{
-		containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)),
-		customopts.WithNewSnapshot(id, containerdImage, snapshotterOpt),
-		containerd.WithSpec(spec, specOpts...),
-		containerd.WithContainerLabels(sandboxLabels),
-		containerd.WithContainerExtension(sandboxMetadataExtension, &sandbox.Metadata),
-		containerd.WithRuntime(ociRuntime.Type, runtimeOpts)}
-
-	container, err := c.client.NewContainer(ctx, id, opts...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create containerd container: %w", err)
-	}
-	defer func() {
-		if retErr != nil {
-			deferCtx, deferCancel := ctrdutil.DeferContext()
-			defer deferCancel()
-			if err := container.Delete(deferCtx, containerd.WithSnapshotCleanup); err != nil {
-				log.G(ctx).WithError(err).Errorf("Failed to delete containerd container %q", id)
-			}
-		}
-	}()
-
-	// Create sandbox container root directories.
-	sandboxRootDir := c.getSandboxRootDir(id)
-	if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
-		return nil, fmt.Errorf("failed to create sandbox root directory %q: %w",
-			sandboxRootDir, err)
-	}
-	defer func() {
-		if retErr != nil {
-			// Cleanup the sandbox root directory.
-			if err := c.os.RemoveAll(sandboxRootDir); err != nil {
-				log.G(ctx).WithError(err).Errorf("Failed to remove sandbox root directory %q",
-					sandboxRootDir)
-			}
-		}
-	}()
-	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
-	if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
-		return nil, fmt.Errorf("failed to create volatile sandbox root directory %q: %w",
-			volatileSandboxRootDir, err)
-	}
-	defer func() {
-		if retErr != nil {
-			// Cleanup the volatile sandbox root directory.
-			if err := c.os.RemoveAll(volatileSandboxRootDir); err != nil {
-				log.G(ctx).WithError(err).Errorf("Failed to remove volatile sandbox root directory %q",
-					volatileSandboxRootDir)
-			}
-		}
-	}()
-
-	// Setup files required for the sandbox.
-	if err = c.setupSandboxFiles(id, config); err != nil {
-		return nil, fmt.Errorf("failed to setup sandbox files: %w", err)
-	}
-	defer func() {
-		if retErr != nil {
-			if err = c.cleanupSandboxFiles(id, config); err != nil {
-				log.G(ctx).WithError(err).Errorf("Failed to cleanup sandbox files in %q",
-					sandboxRootDir)
-			}
-		}
-	}()
-
-	// Update sandbox created timestamp.
-	info, err := container.Info(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get sandbox container info: %w", err)
-	}
-
-	// Create sandbox task in containerd.
-	log.G(ctx).Tracef("Create sandbox container (id=%q, name=%q).",
-		id, name)
-
-	taskOpts := c.taskOpts(ociRuntime.Type)
-	if ociRuntime.Path != "" {
-		taskOpts = append(taskOpts, containerd.WithRuntimePath(ociRuntime.Path))
-	}
-	// We don't need stdio for sandbox container.
-	task, err := container.NewTask(ctx, containerdio.NullIO, taskOpts...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create containerd task: %w", err)
-	}
-	defer func() {
-		if retErr != nil {
-			deferCtx, deferCancel := ctrdutil.DeferContext()
-			defer deferCancel()
-			// Cleanup the sandbox container if an error is returned.
-			if _, err := task.Delete(deferCtx, WithNRISandboxDelete(id), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
-				log.G(ctx).WithError(err).Errorf("Failed to delete sandbox container %q", id)
-			}
-		}
-	}()
-
-	// wait is a long running background request, no timeout needed.
-	exitCh, err := task.Wait(ctrdutil.NamespacedContext())
-	if err != nil {
-		return nil, fmt.Errorf("failed to wait for sandbox container task: %w", err)
-	}
-
-	nric, err := nri.New()
-	if err != nil {
-		return nil, fmt.Errorf("unable to create nri client: %w", err)
-	}
-	if nric != nil {
-		nriSB := &nri.Sandbox{
-			ID:     id,
-			Labels: config.Labels,
-		}
-		if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil {
-			return nil, fmt.Errorf("nri invoke: %w", err)
-		}
-	}
-
-	if err := task.Start(ctx); err != nil {
-		return nil, fmt.Errorf("failed to start sandbox container task %q: %w", id, err)
-	}
 
 	if err := sandbox.Status.Update(func(status sandboxstore.Status) (sandboxstore.Status, error) {
 		// Set the pod sandbox as ready after successfully start sandbox container.
-		status.Pid = task.Pid()
+		status.Pid = resp.Pid
 		status.State = sandboxstore.StateReady
-		status.CreatedAt = info.CreatedAt
+		status.CreatedAt = protobuf.FromTimestamp(resp.CreatedAt)
 		return status, nil
 	}); err != nil {
 		return nil, fmt.Errorf("failed to update sandbox status: %w", err)
 	}
 
+	// TODO: get rid of this. sandbox object should no longer have Container field.
+	container, err := c.client.LoadContainer(ctx, id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load container %q for sandbox: %w", id, err)
+	}
 	// Add sandbox into sandbox store in INIT state.
 	sandbox.Container = container
 
@@ -346,14 +203,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, fmt.Errorf("failed to add sandbox %+v into store: %w", sandbox, err)
 	}
 
-	// start the monitor after adding sandbox into the store, this ensures
+	sandboxRuntimeCreateTimer.WithValues(labels["oci_runtime_type"]).UpdateSince(runtimeStart)
-	// that sandbox is in the store, when event monitor receives the TaskExit event.
-	//
-	// TaskOOM from containerd may come before sandbox is added to store,
-	// but we don't care about sandbox TaskOOM right now, so it is fine.
-	c.eventMonitor.startSandboxExitMonitor(context.Background(), id, task.Pid(), exitCh)
-
-	sandboxRuntimeCreateTimer.WithValues(ociRuntime.Type).UpdateSince(runtimeStart)
 
 	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
 }

pkg/cri/sbserver/sandbox_run_linux.go

@@ -19,305 +19,14 @@ package sbserver
 import (
 	"fmt"
 	"os"
-	"strconv"
-	"strings"
 
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/plugin"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/selinux/go-selinux"
-	"golang.org/x/sys/unix"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
-	"github.com/containerd/containerd/pkg/cri/annotations"
-	customopts "github.com/containerd/containerd/pkg/cri/opts"
 	osinterface "github.com/containerd/containerd/pkg/os"
-	"github.com/containerd/containerd/pkg/userns"
 )
 
-func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
-	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
-	// Creates a spec Generator with the default spec.
-	// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
-	specOpts := []oci.SpecOpts{
-		oci.WithoutRunMount,
-		customopts.WithoutDefaultSecuritySettings,
-		customopts.WithRelativeRoot(relativeRootfsPath),
-		oci.WithEnv(imageConfig.Env),
-		oci.WithRootFSReadonly(),
-		oci.WithHostname(config.GetHostname()),
-	}
-	if imageConfig.WorkingDir != "" {
-		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
-	}
-
-	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
-		// Pause image must have entrypoint or cmd.
-		return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
-	}
-	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
-
-	// Set cgroups parent.
-	if c.config.DisableCgroup {
-		specOpts = append(specOpts, customopts.WithDisabledCgroups)
-	} else {
-		if config.GetLinux().GetCgroupParent() != "" {
-			cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id)
-			specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
-		}
-	}
-
-	// When cgroup parent is not set, containerd-shim will create container in a child cgroup
-	// of the cgroup itself is in.
-	// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
-
-	// Set namespace options.
-	var (
-		securityContext = config.GetLinux().GetSecurityContext()
-		nsOptions       = securityContext.GetNamespaceOptions()
-	)
-	if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE {
-		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace))
-		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.UTSNamespace))
-	} else {
-		specOpts = append(specOpts, oci.WithLinuxNamespace(
-			runtimespec.LinuxNamespace{
-				Type: runtimespec.NetworkNamespace,
-				Path: nsPath,
-			}))
-	}
-	if nsOptions.GetPid() == runtime.NamespaceMode_NODE {
-		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace))
-	}
-	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
-		specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace))
-	}
-
-	// It's fine to generate the spec before the sandbox /dev/shm
-	// is actually created.
-	sandboxDevShm := c.getSandboxDevShm(id)
-	if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
-		sandboxDevShm = devShm
-	}
-	// Remove the default /dev/shm mount from defaultMounts, it is added in oci/mounts.go.
-	specOpts = append(specOpts, oci.WithoutMounts(devShm))
-	// In future the when user-namespace is enabled, the `nosuid, nodev, noexec` flags are
-	// required, otherwise the remount will fail with EPERM. Just use them unconditionally,
-	// they are nice to have anyways.
-	specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{
-		{
-			Source:      sandboxDevShm,
-			Destination: devShm,
-			Type:        "bind",
-			Options:     []string{"rbind", "ro", "nosuid", "nodev", "noexec"},
-		},
-		// Add resolv.conf for katacontainers to setup the DNS of pod VM properly.
-		{
-			Source:      c.getResolvPath(id),
-			Destination: resolvConfPath,
-			Type:        "bind",
-			Options:     []string{"rbind", "ro"},
-		},
-	}))
-
-	processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions())
-	if err != nil {
-		return nil, fmt.Errorf("failed to init selinux options %+v: %w", securityContext.GetSelinuxOptions(), err)
-	}
-	defer func() {
-		if retErr != nil {
-			selinux.ReleaseLabel(processLabel)
-		}
-	}()
-
-	supplementalGroups := securityContext.GetSupplementalGroups()
-	specOpts = append(specOpts,
-		customopts.WithSelinuxLabels(processLabel, mountLabel),
-		customopts.WithSupplementalGroups(supplementalGroups),
-	)
-
-	// Add sysctls
-	sysctls := config.GetLinux().GetSysctls()
-	if sysctls == nil {
-		sysctls = make(map[string]string)
-	}
-	_, ipUnprivilegedPortStart := sysctls["net.ipv4.ip_unprivileged_port_start"]
-	_, pingGroupRange := sysctls["net.ipv4.ping_group_range"]
-	if nsOptions.GetNetwork() != runtime.NamespaceMode_NODE {
-		if c.config.EnableUnprivilegedPorts && !ipUnprivilegedPortStart {
-			sysctls["net.ipv4.ip_unprivileged_port_start"] = "0"
-		}
-		if c.config.EnableUnprivilegedICMP && !pingGroupRange && !userns.RunningInUserNS() {
-			sysctls["net.ipv4.ping_group_range"] = "0 2147483647"
-		}
-	}
-	specOpts = append(specOpts, customopts.WithSysctls(sysctls))
-
-	// Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile
-
-	if !c.config.DisableCgroup {
-		specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
-	}
-
-	if res := config.GetLinux().GetResources(); res != nil {
-		specOpts = append(specOpts,
-			customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)),
-			customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)),
-			customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)),
-			customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10)))
-	}
-
-	specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
-
-	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
-		runtimePodAnnotations) {
-		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
-	}
-
-	specOpts = append(specOpts,
-		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
-		customopts.WithAnnotation(annotations.SandboxID, id),
-		customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
-		customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
-		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
-	)
-
-	return c.runtimeSpec(id, "", specOpts...)
-}
-
-// sandboxContainerSpecOpts generates OCI spec options for
-// the sandbox container.
-func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
-	var (
-		securityContext = config.GetLinux().GetSecurityContext()
-		specOpts        []oci.SpecOpts
-		err             error
-	)
-	ssp := securityContext.GetSeccomp()
-	if ssp == nil {
-		ssp, err = generateSeccompSecurityProfile(
-			securityContext.GetSeccompProfilePath(), //nolint:staticcheck // Deprecated but we don't want to remove yet
-			c.config.UnsetSeccompProfile)
-		if err != nil {
-			return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
-		}
-	}
-	seccompSpecOpts, err := c.generateSeccompSpecOpts(
-		ssp,
-		securityContext.GetPrivileged(),
-		c.seccompEnabled())
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
-	}
-	if seccompSpecOpts != nil {
-		specOpts = append(specOpts, seccompSpecOpts)
-	}
-
-	userstr, err := generateUserString(
-		"",
-		securityContext.GetRunAsUser(),
-		securityContext.GetRunAsGroup(),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate user string: %w", err)
-	}
-	if userstr == "" {
-		// Lastly, since no user override was passed via CRI try to set via OCI
-		// Image
-		userstr = imageConfig.User
-	}
-	if userstr != "" {
-		specOpts = append(specOpts, oci.WithUser(userstr))
-	}
-	return specOpts, nil
-}
-
-// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
-// /etc/resolv.conf and /etc/hostname.
-func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
-	sandboxEtcHostname := c.getSandboxHostname(id)
-	hostname := config.GetHostname()
-	if hostname == "" {
-		var err error
-		hostname, err = c.os.Hostname()
-		if err != nil {
-			return fmt.Errorf("failed to get hostname: %w", err)
-		}
-	}
-	if err := c.os.WriteFile(sandboxEtcHostname, []byte(hostname+"\n"), 0644); err != nil {
-		return fmt.Errorf("failed to write hostname to %q: %w", sandboxEtcHostname, err)
-	}
-
-	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
-	sandboxEtcHosts := c.getSandboxHosts(id)
-	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
-		return fmt.Errorf("failed to generate sandbox hosts file %q: %w", sandboxEtcHosts, err)
-	}
-
-	// Set DNS options. Maintain a resolv.conf for the sandbox.
-	var err error
-	resolvContent := ""
-	if dnsConfig := config.GetDnsConfig(); dnsConfig != nil {
-		resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options)
-		if err != nil {
-			return fmt.Errorf("failed to parse sandbox DNSConfig %+v: %w", dnsConfig, err)
-		}
-	}
-	resolvPath := c.getResolvPath(id)
-	if resolvContent == "" {
-		// copy host's resolv.conf to resolvPath
-		err = c.os.CopyFile(resolvConfPath, resolvPath, 0644)
-		if err != nil {
-			return fmt.Errorf("failed to copy host's resolv.conf to %q: %w", resolvPath, err)
-		}
-	} else {
-		err = c.os.WriteFile(resolvPath, []byte(resolvContent), 0644)
-		if err != nil {
-			return fmt.Errorf("failed to write resolv content to %q: %w", resolvPath, err)
-		}
-	}
-
-	// Setup sandbox /dev/shm.
-	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
-		if _, err := c.os.Stat(devShm); err != nil {
-			return fmt.Errorf("host %q is not available for host ipc: %w", devShm, err)
-		}
-	} else {
-		sandboxDevShm := c.getSandboxDevShm(id)
-		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
-			return fmt.Errorf("failed to create sandbox shm: %w", err)
-		}
-		shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
-		if err := c.os.(osinterface.UNIX).Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
-			return fmt.Errorf("failed to mount sandbox shm: %w", err)
-		}
-	}
-
-	return nil
-}
-
-// parseDNSOptions parse DNS options into resolv.conf format content,
-// if none option is specified, will return empty with no error.
-func parseDNSOptions(servers, searches, options []string) (string, error) {
-	resolvContent := ""
-
-	if len(searches) > 0 {
-		resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " "))
-	}
-
-	if len(servers) > 0 {
-		resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver "))
-	}
-
-	if len(options) > 0 {
-		resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " "))
-	}
-
-	return resolvContent, nil
-}
-
 // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
 // remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
 func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {

pkg/cri/sbserver/sandbox_run_other.go

@@ -21,29 +21,9 @@ package sbserver
 
 import (
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/oci"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
-func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
-	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
-	return c.runtimeSpec(id, "")
-}
-
-// sandboxContainerSpecOpts generates OCI spec options for
-// the sandbox container.
-func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
-	return []oci.SpecOpts{}, nil
-}
-
-// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
-// /etc/resolv.conf and /etc/hostname.
-func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
-	return nil
-}
-
 // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
 // remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
 func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {

pkg/cri/sbserver/sandbox_run_test.go

@@ -19,152 +19,13 @@ package sbserver
 import (
 	"context"
 	"net"
-	goruntime "runtime"
 	"testing"
 
 	"github.com/containerd/go-cni"
-	"github.com/containerd/typeurl"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	"github.com/containerd/containerd/pkg/cri/annotations"
-	criconfig "github.com/containerd/containerd/pkg/cri/config"
-	sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
 )
 
-func TestSandboxContainerSpec(t *testing.T) {
-	switch goruntime.GOOS {
-	case "darwin":
-		t.Skip("not implemented on Darwin")
-	case "freebsd":
-		t.Skip("not implemented on FreeBSD")
-	}
-	testID := "test-id"
-	nsPath := "test-cni"
-	for desc, test := range map[string]struct {
-		configChange      func(*runtime.PodSandboxConfig)
-		podAnnotations    []string
-		imageConfigChange func(*imagespec.ImageConfig)
-		specCheck         func(*testing.T, *runtimespec.Spec)
-		expectErr         bool
-	}{
-		"should return error when entrypoint and cmd are empty": {
-			imageConfigChange: func(c *imagespec.ImageConfig) {
-				c.Entrypoint = nil
-				c.Cmd = nil
-			},
-			expectErr: true,
-		},
-		"a passthrough annotation should be passed as an OCI annotation": {
-			podAnnotations: []string{"c"},
-			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
-				assert.Equal(t, spec.Annotations["c"], "d")
-			},
-		},
-		"a non-passthrough annotation should not be passed as an OCI annotation": {
-			configChange: func(c *runtime.PodSandboxConfig) {
-				c.Annotations["d"] = "e"
-			},
-			podAnnotations: []string{"c"},
-			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
-				assert.Equal(t, spec.Annotations["c"], "d")
-				_, ok := spec.Annotations["d"]
-				assert.False(t, ok)
-			},
-		},
-		"passthrough annotations should support wildcard match": {
-			configChange: func(c *runtime.PodSandboxConfig) {
-				c.Annotations["t.f"] = "j"
-				c.Annotations["z.g"] = "o"
-				c.Annotations["z"] = "o"
-				c.Annotations["y.ca"] = "b"
-				c.Annotations["y"] = "b"
-			},
-			podAnnotations: []string{"t*", "z.*", "y.c*"},
-			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
-				assert.Equal(t, spec.Annotations["t.f"], "j")
-				assert.Equal(t, spec.Annotations["z.g"], "o")
-				assert.Equal(t, spec.Annotations["y.ca"], "b")
-				_, ok := spec.Annotations["y"]
-				assert.False(t, ok)
-				_, ok = spec.Annotations["z"]
-				assert.False(t, ok)
-			},
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			c := newTestCRIService()
-			config, imageConfig, specCheck := getRunPodSandboxTestData()
-			if test.configChange != nil {
-				test.configChange(config)
-			}
-
-			if test.imageConfigChange != nil {
-				test.imageConfigChange(imageConfig)
-			}
-			spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath,
-				test.podAnnotations)
-			if test.expectErr {
-				assert.Error(t, err)
-				assert.Nil(t, spec)
-				return
-			}
-			assert.NoError(t, err)
-			assert.NotNil(t, spec)
-			specCheck(t, testID, spec)
-			if test.specCheck != nil {
-				test.specCheck(t, spec)
-			}
-		})
-	}
-}
-
-func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
-	for desc, test := range map[string]struct {
-		configChange func(*runtime.PodSandboxConfig)
-	}{
-		"should marshal original config": {},
-		"should marshal Linux": {
-			configChange: func(c *runtime.PodSandboxConfig) {
-				if c.Linux == nil {
-					c.Linux = &runtime.LinuxPodSandboxConfig{}
-				}
-				c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
-					NamespaceOptions: &runtime.NamespaceOption{
-						Network: runtime.NamespaceMode_NODE,
-						Pid:     runtime.NamespaceMode_NODE,
-						Ipc:     runtime.NamespaceMode_NODE,
-					},
-					SupplementalGroups: []int64{1111, 2222},
-				}
-			},
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			meta := &sandboxstore.Metadata{
-				ID:        "1",
-				Name:      "sandbox_1",
-				NetNSPath: "/home/cloud",
-			}
-			meta.Config, _, _ = getRunPodSandboxTestData()
-			if test.configChange != nil {
-				test.configChange(meta.Config)
-			}
-
-			any, err := typeurl.MarshalAny(meta)
-			assert.NoError(t, err)
-			data, err := typeurl.UnmarshalAny(any)
-			assert.NoError(t, err)
-			assert.IsType(t, &sandboxstore.Metadata{}, data)
-			curMeta, ok := data.(*sandboxstore.Metadata)
-			assert.True(t, ok)
-			assert.Equal(t, meta, curMeta)
-		})
-	}
-}
-
 func TestToCNIPortMappings(t *testing.T) {
 	for desc, test := range map[string]struct {
 		criPortMappings []*runtime.PortMapping
@@ -315,212 +176,3 @@ func TestSelectPodIP(t *testing.T) {
 		})
 	}
 }
-
-func TestHostAccessingSandbox(t *testing.T) {
-	privilegedContext := &runtime.PodSandboxConfig{
-		Linux: &runtime.LinuxPodSandboxConfig{
-			SecurityContext: &runtime.LinuxSandboxSecurityContext{
-				Privileged: true,
-			},
-		},
-	}
-	nonPrivilegedContext := &runtime.PodSandboxConfig{
-		Linux: &runtime.LinuxPodSandboxConfig{
-			SecurityContext: &runtime.LinuxSandboxSecurityContext{
-				Privileged: false,
-			},
-		},
-	}
-	hostNamespace := &runtime.PodSandboxConfig{
-		Linux: &runtime.LinuxPodSandboxConfig{
-			SecurityContext: &runtime.LinuxSandboxSecurityContext{
-				Privileged: false,
-				NamespaceOptions: &runtime.NamespaceOption{
-					Network: runtime.NamespaceMode_NODE,
-					Pid:     runtime.NamespaceMode_NODE,
-					Ipc:     runtime.NamespaceMode_NODE,
-				},
-			},
-		},
-	}
-	tests := []struct {
-		name   string
-		config *runtime.PodSandboxConfig
-		want   bool
-	}{
-		{"Security Context is nil", nil, false},
-		{"Security Context is privileged", privilegedContext, false},
-		{"Security Context is not privileged", nonPrivilegedContext, false},
-		{"Security Context namespace host access", hostNamespace, true},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := hostAccessingSandbox(tt.config); got != tt.want {
-				t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestGetSandboxRuntime(t *testing.T) {
-	untrustedWorkloadRuntime := criconfig.Runtime{
-		Type:   "io.containerd.runtime.v1.linux",
-		Engine: "untrusted-workload-runtime",
-		Root:   "",
-	}
-
-	defaultRuntime := criconfig.Runtime{
-		Type:   "io.containerd.runtime.v1.linux",
-		Engine: "default-runtime",
-		Root:   "",
-	}
-
-	fooRuntime := criconfig.Runtime{
-		Type:   "io.containerd.runtime.v1.linux",
-		Engine: "foo-bar",
-		Root:   "",
-	}
-
-	for desc, test := range map[string]struct {
-		sandboxConfig   *runtime.PodSandboxConfig
-		runtimeHandler  string
-		runtimes        map[string]criconfig.Runtime
-		expectErr       bool
-		expectedRuntime criconfig.Runtime
-	}{
-		"should return error if untrusted workload requires host access": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Linux: &runtime.LinuxPodSandboxConfig{
-					SecurityContext: &runtime.LinuxSandboxSecurityContext{
-						Privileged: false,
-						NamespaceOptions: &runtime.NamespaceOption{
-							Network: runtime.NamespaceMode_NODE,
-							Pid:     runtime.NamespaceMode_NODE,
-							Ipc:     runtime.NamespaceMode_NODE,
-						},
-					},
-				},
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-			},
-			expectErr: true,
-		},
-		"should use untrusted workload runtime for untrusted workload": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-			},
-			expectedRuntime: untrustedWorkloadRuntime,
-		},
-		"should use default runtime for regular workload": {
-			sandboxConfig: &runtime.PodSandboxConfig{},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault: defaultRuntime,
-			},
-			expectedRuntime: defaultRuntime,
-		},
-		"should use default runtime for trusted workload": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "false",
-				},
-			},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-			},
-			expectedRuntime: defaultRuntime,
-		},
-		"should return error if untrusted workload runtime is required but not configured": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault: defaultRuntime,
-			},
-			expectErr: true,
-		},
-		"should use 'untrusted' runtime for untrusted workload": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-			},
-			expectedRuntime: untrustedWorkloadRuntime,
-		},
-		"should use 'untrusted' runtime for untrusted workload & handler": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimeHandler: "untrusted",
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-			},
-			expectedRuntime: untrustedWorkloadRuntime,
-		},
-		"should return an error if untrusted annotation with conflicting handler": {
-			sandboxConfig: &runtime.PodSandboxConfig{
-				Annotations: map[string]string{
-					annotations.UntrustedWorkload: "true",
-				},
-			},
-			runtimeHandler: "foo",
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-				"foo":                      fooRuntime,
-			},
-			expectErr: true,
-		},
-		"should use correct runtime for a runtime handler": {
-			sandboxConfig:  &runtime.PodSandboxConfig{},
-			runtimeHandler: "foo",
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault:   defaultRuntime,
-				criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
-				"foo":                      fooRuntime,
-			},
-			expectedRuntime: fooRuntime,
-		},
-		"should return error if runtime handler is required but not configured": {
-			sandboxConfig:  &runtime.PodSandboxConfig{},
-			runtimeHandler: "bar",
-			runtimes: map[string]criconfig.Runtime{
-				criconfig.RuntimeDefault: defaultRuntime,
-				"foo":                    fooRuntime,
-			},
-			expectErr: true,
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			cri := newTestCRIService()
-			cri.config = criconfig.Config{
-				PluginConfig: criconfig.DefaultConfig(),
-			}
-			cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
-			cri.config.ContainerdConfig.Runtimes = test.runtimes
-			r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
-			assert.Equal(t, test.expectErr, err != nil)
-			assert.Equal(t, test.expectedRuntime, r)
-		})
-	}
-}

pkg/cri/sbserver/sandbox_run_windows.go

@@ -17,91 +17,10 @@
 package sbserver
 
 import (
-	"fmt"
-	"strconv"
-
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/oci"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	"github.com/containerd/containerd/pkg/cri/annotations"
-	customopts "github.com/containerd/containerd/pkg/cri/opts"
 )
 
-func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
-	imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
-	// Creates a spec Generator with the default spec.
-	specOpts := []oci.SpecOpts{
-		oci.WithEnv(imageConfig.Env),
-		oci.WithHostname(config.GetHostname()),
-	}
-	if imageConfig.WorkingDir != "" {
-		specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
-	}
-
-	if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
-		// Pause image must have entrypoint or cmd.
-		return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
-	}
-	specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
-
-	specOpts = append(specOpts,
-		// Clear the root location since hcsshim expects it.
-		// NOTE: readonly rootfs doesn't work on windows.
-		customopts.WithoutRoot,
-		customopts.WithWindowsNetworkNamespace(nsPath),
-	)
-
-	specOpts = append(specOpts, customopts.WithWindowsDefaultSandboxShares)
-
-	// Start with the image config user and override below if RunAsUsername is not "".
-	username := imageConfig.User
-
-	runAsUser := config.GetWindows().GetSecurityContext().GetRunAsUsername()
-	if runAsUser != "" {
-		username = runAsUser
-	}
-
-	cs := config.GetWindows().GetSecurityContext().GetCredentialSpec()
-	if cs != "" {
-		specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs))
-	}
-
-	// There really isn't a good Windows way to verify that the username is available in the
-	// image as early as here like there is for Linux. Later on in the stack hcsshim
-	// will handle the behavior of erroring out if the user isn't available in the image
-	// when trying to run the init process.
-	specOpts = append(specOpts, oci.WithUser(username))
-
-	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
-		runtimePodAnnotations) {
-		specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
-	}
-
-	specOpts = append(specOpts,
-		customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
-		customopts.WithAnnotation(annotations.SandboxID, id),
-		customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
-		customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
-		customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
-		customopts.WithAnnotation(annotations.WindowsHostProcess, strconv.FormatBool(config.GetWindows().GetSecurityContext().GetHostProcess())),
-	)
-
-	return c.runtimeSpec(id, "", specOpts...)
-}
-
-// No sandbox container spec options for windows yet.
-func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
-	return nil, nil
-}
-
-// No sandbox files needed for windows.
-func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
-	return nil
-}
-
 // No sandbox files needed for windows.
 func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
 	return nil

pkg/cri/sbserver/service.go

@@ -17,6 +17,7 @@
 package sbserver
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -28,9 +29,11 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/pkg/cri/sbserver/podsandbox"
 	"github.com/containerd/containerd/pkg/cri/streaming"
 	"github.com/containerd/containerd/pkg/kmutex"
 	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/sandbox"
 	"github.com/containerd/go-cni"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -88,6 +91,8 @@ type criService struct {
 	sandboxNameIndex *registrar.Registrar
 	// containerStore stores all resources associated with containers.
 	containerStore *containerstore.Store
+	// sandboxController controls sandbox lifecycle (and hides implementation details behind).
+	sandboxController sandbox.Controller
 	// containerNameIndex stores all container names and make sure each
 	// name is unique.
 	containerNameIndex *registrar.Registrar
@@ -181,9 +186,17 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
 		return nil, err
 	}
 
+	c.sandboxController = podsandbox.New(config, client, c.sandboxStore, c.os, c, c.baseOCISpecs)
+
 	return c, nil
 }
 
+// StartSandboxExitMonitor is a temporary workaround to call monitor from pause controller.
+// TODO: get rid of this.
+func (c *criService) StartSandboxExitMonitor(ctx context.Context, id string, pid uint32, exitCh <-chan containerd.ExitStatus) <-chan struct{} {
+	return c.eventMonitor.startSandboxExitMonitor(ctx, id, pid, exitCh)
+}
+
 // Register registers all required services onto a specific grpc server.
 // This is used by containerd cri plugin.
 func (c *criService) Register(s *grpc.Server) error {

sandbox.go

@@ -82,12 +82,12 @@ func (s *sandboxClient) Labels(ctx context.Context) (map[string]string, error) {
 }
 
 func (s *sandboxClient) Start(ctx context.Context) error {
-	pid, err := s.client.SandboxController().Start(ctx, s.ID())
+	resp, err := s.client.SandboxController().Start(ctx, s.ID())
 	if err != nil {
 		return err
 	}
 
-	s.pid = &pid
+	s.pid = &resp.Pid
 	return nil
 }
 
@@ -115,10 +115,16 @@ func (s *sandboxClient) Wait(ctx context.Context) (<-chan ExitStatus, error) {
 }
 
 func (s *sandboxClient) Stop(ctx context.Context) error {
-	return s.client.SandboxController().Shutdown(ctx, s.ID())
+	if _, err := s.client.SandboxController().Stop(ctx, s.ID()); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (s *sandboxClient) Delete(ctx context.Context) error {
+	if _, err := s.client.SandboxController().Delete(ctx, s.ID()); err != nil {
+		return err
+	}
 	return s.client.SandboxStore().Delete(ctx, s.ID())
 }
 

sandbox/controller.go

@@ -30,12 +30,14 @@ type Controller interface {
 	// containerd will run new shim runtime instance and will invoke Start to create a sandbox process.
 	// This routine must be invoked before scheduling containers on this instance.
 	// Once started clients may run containers via Task service (additionally specifying sandbox id the container will belong to).
-	Start(ctx context.Context, sandboxID string) (uint32, error)
+	Start(ctx context.Context, sandboxID string) (*sandbox.ControllerStartResponse, error)
-	// Shutdown deletes and cleans all tasks and sandbox instance.
+	// Stop will stop sandbox intances
-	Shutdown(ctx context.Context, sandboxID string) error
+	Stop(ctx context.Context, sandboxID string) (*sandbox.ControllerStopResponse, error)
 	// Wait blocks until sandbox process exits.
 	Wait(ctx context.Context, sandboxID string) (*sandbox.ControllerWaitResponse, error)
 	// Status will query sandbox process status. It is heavier than Ping call and must be used whenever you need to
 	// gather metadata about current sandbox state (status, uptime, resource use, etc).
 	Status(ctx context.Context, sandboxID string) (*sandbox.ControllerStatusResponse, error)
+	// Delete deletes and cleans all tasks and sandbox instance.
+	Delete(ctx context.Context, sandboxID string) (*sandbox.ControllerDeleteResponse, error)
 }

sandbox/store.go

@@ -18,8 +18,10 @@ package sandbox
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/typeurl"
 )
 
@@ -64,3 +66,51 @@ type Store interface {
 	// Delete a sandbox from metadata store using the id
 	Delete(ctx context.Context, id string) error
 }
+
+// AddExtension is a helper function to add sandbox metadata extension.
+func (s *Sandbox) AddExtension(name string, obj interface{}) error {
+	if s.Extensions == nil {
+		s.Extensions = map[string]typeurl.Any{}
+	}
+
+	out, err := typeurl.MarshalAny(obj)
+	if err != nil {
+		return fmt.Errorf("failed to marshal sandbox extension %q: %w", name, err)
+	}
+
+	s.Extensions[name] = out
+	return nil
+}
+
+// AddLabel adds a label to sandbox's labels.
+func (s *Sandbox) AddLabel(name string, value string) {
+	if s.Labels == nil {
+		s.Labels = map[string]string{}
+	}
+
+	s.Labels[name] = value
+}
+
+// GetExtension retrieves a sandbox extension by name.
+func (s *Sandbox) GetExtension(name string, obj interface{}) error {
+	out, ok := s.Extensions[name]
+	if !ok {
+		return errdefs.ErrNotFound
+	}
+
+	if err := typeurl.UnmarshalTo(out, obj); err != nil {
+		return fmt.Errorf("failed to unmarshal sandbox extension %q: %w", name, err)
+	}
+
+	return nil
+}
+
+// GetLabel retrieves a sandbox label by name.
+func (s *Sandbox) GetLabel(name string) (string, error) {
+	out, ok := s.Labels[name]
+	if !ok {
+		return "", fmt.Errorf("unable to find label %q in sandbox metadata: %w", name, errdefs.ErrNotFound)
+	}
+
+	return out, nil
+}

sandbox/store_test.go

@@ -0,0 +1,40 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sandbox
+
+import (
+	"testing"
+
+	"github.com/containerd/typeurl"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAddExtension(t *testing.T) {
+	sb := Sandbox{ID: "1"}
+
+	type test struct{ Name string }
+
+	typeurl.Register(&test{})
+
+	var in = test{Name: "test"}
+	assert.NoError(t, sb.AddExtension("test", &in))
+
+	var out test
+	err := sb.GetExtension("test", &out)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", out.Name)
+}

sandbox_controller.go

@@ -36,22 +36,31 @@ func NewSandboxRemoteController(client api.ControllerClient) sb.Controller {
 	return &sandboxRemoteController{client: client}
 }
 
-func (s *sandboxRemoteController) Start(ctx context.Context, sandboxID string) (uint32, error) {
+func (s *sandboxRemoteController) Start(ctx context.Context, sandboxID string) (*api.ControllerStartResponse, error) {
 	resp, err := s.client.Start(ctx, &api.ControllerStartRequest{SandboxID: sandboxID})
 	if err != nil {
-		return 0, errdefs.FromGRPC(err)
+		return nil, errdefs.FromGRPC(err)
 	}
 
-	return resp.Pid, nil
+	return resp, nil
 }
 
-func (s *sandboxRemoteController) Shutdown(ctx context.Context, sandboxID string) error {
+func (s *sandboxRemoteController) Stop(ctx context.Context, sandboxID string) (*api.ControllerStopResponse, error) {
-	_, err := s.client.Shutdown(ctx, &api.ControllerShutdownRequest{SandboxID: sandboxID})
+	resp, err := s.client.Stop(ctx, &api.ControllerStopRequest{SandboxID: sandboxID})
 	if err != nil {
-		return errdefs.FromGRPC(err)
+		return nil, errdefs.FromGRPC(err)
 	}
 
-	return nil
+	return resp, nil
+}
+
+func (s *sandboxRemoteController) Delete(ctx context.Context, sandboxID string) (*api.ControllerDeleteResponse, error) {
+	resp, err := s.client.Delete(ctx, &api.ControllerDeleteRequest{SandboxID: sandboxID})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+
+	return resp, nil
 }
 
 func (s *sandboxRemoteController) Wait(ctx context.Context, sandboxID string) (*api.ControllerWaitResponse, error) {

services.go

@@ -24,7 +24,7 @@ import (
 	imagesapi "github.com/containerd/containerd/api/services/images/v1"
 	introspectionapi "github.com/containerd/containerd/api/services/introspection/v1"
 	namespacesapi "github.com/containerd/containerd/api/services/namespaces/v1"
-	sandboxsapi "github.com/containerd/containerd/api/services/sandbox/v1"
+	sandboxapi "github.com/containerd/containerd/api/services/sandbox/v1"
 	"github.com/containerd/containerd/api/services/tasks/v1"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/content"
@@ -165,14 +165,14 @@ func WithIntrospectionService(in introspection.Service) ServicesOpt {
 }
 
 // WithSandboxStore sets the sandbox store.
-func WithSandboxStore(client sandboxsapi.StoreClient) ServicesOpt {
+func WithSandboxStore(client sandboxapi.StoreClient) ServicesOpt {
 	return func(s *services) {
 		s.sandboxStore = NewRemoteSandboxStore(client)
 	}
 }
 
 // WithSandboxController sets the sandbox controller.
-func WithSandboxController(client sandboxsapi.ControllerClient) ServicesOpt {
+func WithSandboxController(client sandboxapi.ControllerClient) ServicesOpt {
 	return func(s *services) {
 		s.sandboxController = NewSandboxRemoteController(client)
 	}
@@ -227,6 +227,12 @@ func WithInMemoryServices(ic *plugin.InitContext) ClientOpt {
 			srv.IntrospectionService: func(s interface{}) ServicesOpt {
 				return WithIntrospectionClient(s.(introspectionapi.IntrospectionClient))
 			},
+			srv.SandboxStoreService: func(s interface{}) ServicesOpt {
+				return WithSandboxStore(s.(sandboxapi.StoreClient))
+			},
+			srv.SandboxControllerService: func(s interface{}) ServicesOpt {
+				return WithSandboxController(s.(sandboxapi.ControllerClient))
+			},
 		} {
 			p := plugins[s]
 			if p == nil {

services/sandbox/controller_local.go

@@ -123,7 +123,7 @@ func (c *controllerLocal) Start(ctx context.Context, in *api.ControllerStartRequ
 	}, nil
 }
 
-func (c *controllerLocal) Shutdown(ctx context.Context, in *api.ControllerShutdownRequest, opts ...grpc.CallOption) (*api.ControllerShutdownResponse, error) {
+func (c *controllerLocal) Stop(ctx context.Context, in *api.ControllerStopRequest, opts ...grpc.CallOption) (*api.ControllerStopResponse, error) {
 	svc, err := c.getSandbox(ctx, in.SandboxID)
 	if err != nil {
 		return nil, err
@@ -136,11 +136,15 @@ func (c *controllerLocal) Shutdown(ctx context.Context, in *api.ControllerShutdo
 		return nil, fmt.Errorf("failed to stop sandbox: %w", err)
 	}
 
+	return &api.ControllerStopResponse{}, nil
+}
+
+func (c *controllerLocal) Delete(ctx context.Context, in *api.ControllerDeleteRequest, opts ...grpc.CallOption) (*api.ControllerDeleteResponse, error) {
 	if err := c.shims.Delete(ctx, in.SandboxID); err != nil {
 		return nil, fmt.Errorf("failed to delete sandbox shim: %w", err)
 	}
 
-	return &api.ControllerShutdownResponse{}, nil
+	return &api.ControllerDeleteResponse{}, nil
 }
 
 func (c *controllerLocal) Wait(ctx context.Context, in *api.ControllerWaitRequest, opts ...grpc.CallOption) (*api.ControllerWaitResponse, error) {

services/sandbox/controller_service.go

@@ -74,9 +74,9 @@ func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartR
 	return s.local.Start(ctx, req)
 }
 
-func (s *controllerService) Shutdown(ctx context.Context, req *api.ControllerShutdownRequest) (*api.ControllerShutdownResponse, error) {
+func (s *controllerService) Stop(ctx context.Context, req *api.ControllerStopRequest) (*api.ControllerStopResponse, error) {
 	log.G(ctx).WithField("req", req).Debug("delete sandbox")
-	return s.local.Shutdown(ctx, req)
+	return s.local.Stop(ctx, req)
 }
 
 func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitRequest) (*api.ControllerWaitResponse, error) {
@@ -88,3 +88,8 @@ func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatu
 	log.G(ctx).WithField("req", req).Debug("sandbox status")
 	return s.local.Status(ctx, req)
 }
+
+func (s *controllerService) Delete(ctx context.Context, req *api.ControllerDeleteRequest) (*api.ControllerDeleteResponse, error) {
+	log.G(ctx).WithField("req", req).Debug("delete sandbox")
+	return s.local.Delete(ctx, req)
+}