diff --git a/api/next.pb.txt b/api/next.pb.txt index 33ffae1ce..b184223e2 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -4974,6 +4974,13 @@ file { type_name: ".containerd.types.Sandbox.ExtensionsEntry" json_name: "extensions" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } nested_type { name: "Runtime" field { @@ -5194,6 +5201,13 @@ file { type_name: ".containerd.services.sandbox.v1.ControllerCreateRequest.AnnotationsEntry" json_name: "annotations" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } nested_type { name: "AnnotationsEntry" field { @@ -5234,6 +5248,13 @@ file { type: TYPE_STRING json_name: "sandboxId" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerStartResponse" @@ -5297,6 +5318,13 @@ file { type: TYPE_STRING json_name: "sandboxId" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerPlatformResponse" @@ -5325,6 +5353,13 @@ file { type: TYPE_UINT32 json_name: "timeoutSecs" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerStopResponse" @@ -5338,6 +5373,13 @@ file { type: TYPE_STRING json_name: "sandboxId" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerWaitResponse" @@ -5373,6 +5415,13 @@ file { type: TYPE_BOOL json_name: "verbose" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerStatusResponse" @@ -5459,6 +5508,13 @@ file { type: TYPE_STRING json_name: "sandboxId" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerShutdownResponse" @@ -5472,6 +5528,13 @@ file { type: TYPE_STRING json_name: "sandboxId" } + field { + name: "sandboxer" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "sandboxer" + } } message_type { name: "ControllerMetricsResponse" diff --git a/api/services/sandbox/v1/sandbox.pb.go b/api/services/sandbox/v1/sandbox.pb.go index 98c40071d..8e74928dc 100644 --- a/api/services/sandbox/v1/sandbox.pb.go +++ b/api/services/sandbox/v1/sandbox.pb.go @@ -525,6 +525,7 @@ type ControllerCreateRequest struct { Options *anypb.Any `protobuf:"bytes,3,opt,name=options,proto3" json:"options,omitempty"` NetnsPath string `protobuf:"bytes,4,opt,name=netns_path,json=netnsPath,proto3" json:"netns_path,omitempty"` Annotations map[string]string `protobuf:"bytes,5,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerCreateRequest) Reset() { @@ -594,6 +595,13 @@ func (x *ControllerCreateRequest) GetAnnotations() map[string]string { return nil } +func (x *ControllerCreateRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerCreateResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -647,6 +655,7 @@ type ControllerStartRequest struct { unknownFields protoimpl.UnknownFields SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerStartRequest) Reset() { @@ -688,6 +697,13 @@ func (x *ControllerStartRequest) GetSandboxID() string { return "" } +func (x *ControllerStartRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerStartResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -765,6 +781,7 @@ type ControllerPlatformRequest struct { unknownFields protoimpl.UnknownFields SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerPlatformRequest) Reset() { @@ -806,6 +823,13 @@ func (x *ControllerPlatformRequest) GetSandboxID() string { return "" } +func (x *ControllerPlatformRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerPlatformResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -860,6 +884,7 @@ type ControllerStopRequest struct { SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` TimeoutSecs uint32 `protobuf:"varint,2,opt,name=timeout_secs,json=timeoutSecs,proto3" json:"timeout_secs,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerStopRequest) Reset() { @@ -908,6 +933,13 @@ func (x *ControllerStopRequest) GetTimeoutSecs() uint32 { return 0 } +func (x *ControllerStopRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerStopResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -952,6 +984,7 @@ type ControllerWaitRequest struct { unknownFields protoimpl.UnknownFields SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerWaitRequest) Reset() { @@ -993,6 +1026,13 @@ func (x *ControllerWaitRequest) GetSandboxID() string { return "" } +func (x *ControllerWaitRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerWaitResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1055,6 +1095,7 @@ type ControllerStatusRequest struct { SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` Verbose bool `protobuf:"varint,2,opt,name=verbose,proto3" json:"verbose,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerStatusRequest) Reset() { @@ -1103,6 +1144,13 @@ func (x *ControllerStatusRequest) GetVerbose() bool { return false } +func (x *ControllerStatusRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerStatusResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1204,6 +1252,7 @@ type ControllerShutdownRequest struct { unknownFields protoimpl.UnknownFields SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerShutdownRequest) Reset() { @@ -1245,6 +1294,13 @@ func (x *ControllerShutdownRequest) GetSandboxID() string { return "" } +func (x *ControllerShutdownRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerShutdownResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1289,6 +1345,7 @@ type ControllerMetricsRequest struct { unknownFields protoimpl.UnknownFields SandboxID string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *ControllerMetricsRequest) Reset() { @@ -1330,6 +1387,13 @@ func (x *ControllerMetricsRequest) GetSandboxID() string { return "" } +func (x *ControllerMetricsRequest) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type ControllerMetricsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1444,7 +1508,7 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x52, 0x07, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x22, 0xe4, 0x02, 0x0a, 0x17, 0x43, + 0x78, 0x52, 0x07, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x22, 0x82, 0x03, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, @@ -1463,103 +1527,118 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x1a, + 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x39, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x16, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, + 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, + 0x72, 0x22, 0x9d, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x39, + 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x5b, 0x0a, 0x06, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x39, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, + 0x01, 0x22, 0x58, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, + 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x54, 0x0a, 0x1a, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, + 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x08, 0x70, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, + 0x6d, 0x22, 0x77, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, + 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x73, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x18, 0x0a, 0x16, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x54, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x37, 0x0a, 0x16, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, - 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x9d, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, - 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, - 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x5b, 0x0a, - 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x43, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, - 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x16, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, + 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x70, + 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x62, + 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, 0x65, 0x72, 0x62, 0x6f, + 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, + 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, + 0x22, 0x92, 0x03, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, + 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x56, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, + 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x39, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, + 0x12, 0x2a, 0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x1a, 0x37, 0x0a, 0x09, + 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3a, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x58, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, - 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x36, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0x59, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, - 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, - 0x63, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x36, 0x0a, 0x15, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, - 0x6f, 0x78, 0x49, 0x64, 0x22, 0x72, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, - 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x52, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, - 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x62, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, 0x65, 0x72, 0x62, 0x6f, 0x73, 0x65, 0x22, 0x92, 0x03, 0x0a, - 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, + 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, + 0x1c, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, + 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x57, 0x0a, + 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x56, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2a, 0x0a, 0x05, - 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, - 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x49, 0x6e, 0x66, 0x6f, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x3a, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, - 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x1c, 0x0a, - 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, - 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x39, 0x0a, 0x18, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, - 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, - 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x22, 0x4f, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, + 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x4f, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, diff --git a/api/services/sandbox/v1/sandbox.proto b/api/services/sandbox/v1/sandbox.proto old mode 100644 new mode 100755 index ae605804c..8b384be4c --- a/api/services/sandbox/v1/sandbox.proto +++ b/api/services/sandbox/v1/sandbox.proto @@ -103,6 +103,7 @@ message ControllerCreateRequest { google.protobuf.Any options = 3; string netns_path = 4; map annotations = 5; + string sandboxer = 10; } message ControllerCreateResponse { @@ -111,6 +112,7 @@ message ControllerCreateResponse { message ControllerStartRequest { string sandbox_id = 1; + string sandboxer = 10; } message ControllerStartResponse { @@ -122,6 +124,7 @@ message ControllerStartResponse { message ControllerPlatformRequest { string sandbox_id = 1; + string sandboxer = 10; } message ControllerPlatformResponse { @@ -131,12 +134,14 @@ message ControllerPlatformResponse { message ControllerStopRequest { string sandbox_id = 1; uint32 timeout_secs = 2; + string sandboxer = 10; } message ControllerStopResponse {} message ControllerWaitRequest { string sandbox_id = 1; + string sandboxer = 10; } message ControllerWaitResponse { @@ -147,6 +152,7 @@ message ControllerWaitResponse { message ControllerStatusRequest { string sandbox_id = 1; bool verbose = 2; + string sandboxer = 10; } message ControllerStatusResponse { @@ -161,14 +167,16 @@ message ControllerStatusResponse { message ControllerShutdownRequest { string sandbox_id = 1; + string sandboxer = 10; } message ControllerShutdownResponse {} message ControllerMetricsRequest { string sandbox_id = 1; + string sandboxer = 10; } message ControllerMetricsResponse { types.Metric metrics = 1; -} \ No newline at end of file +} diff --git a/api/types/sandbox.pb.go b/api/types/sandbox.pb.go index 67594f416..77888bf33 100644 --- a/api/types/sandbox.pb.go +++ b/api/types/sandbox.pb.go @@ -59,6 +59,8 @@ type Sandbox struct { UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` // Extensions allow clients to provide optional blobs that can be handled by runtime. Extensions map[string]*anypb.Any `protobuf:"bytes,7,rep,name=extensions,proto3" json:"extensions,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Sandboxer is the name of the sandbox controller who manages the sandbox. + Sandboxer string `protobuf:"bytes,10,opt,name=sandboxer,proto3" json:"sandboxer,omitempty"` } func (x *Sandbox) Reset() { @@ -142,6 +144,13 @@ func (x *Sandbox) GetExtensions() map[string]*anypb.Any { return nil } +func (x *Sandbox) GetSandboxer() string { + if x != nil { + return x.Sandboxer + } + return "" +} + type Sandbox_Runtime struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -211,7 +220,7 @@ var file_github_com_containerd_containerd_api_types_sandbox_proto_rawDesc = []by 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xee, 0x04, 0x0a, 0x07, 0x53, 0x61, 0x6e, + 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8c, 0x05, 0x0a, 0x07, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x3b, 0x0a, 0x07, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, @@ -236,25 +245,27 @@ var file_github_com_containerd_containerd_api_types_sandbox_proto_rawDesc = []by 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, - 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x4d, 0x0a, 0x07, 0x52, 0x75, - 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, - 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x53, 0x0a, 0x0f, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, - 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x3b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x1a, 0x4d, 0x0a, 0x07, 0x52, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x07, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x53, 0x0a, 0x0f, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x3b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/api/types/sandbox.proto b/api/types/sandbox.proto index b60770619..b0bf233b9 100644 --- a/api/types/sandbox.proto +++ b/api/types/sandbox.proto @@ -41,11 +41,14 @@ message Sandbox { // bundle directory (similary to OCI spec). google.protobuf.Any spec = 3; // Labels provides an area to include arbitrary data on containers. - map labels = 4; + map labels = 4; // CreatedAt is the time the container was first created. google.protobuf.Timestamp created_at = 5; // UpdatedAt is the last time the container was mutated. google.protobuf.Timestamp updated_at = 6; // Extensions allow clients to provide optional blobs that can be handled by runtime. map extensions = 7; + // Sandboxer is the name of the sandbox controller who manages the sandbox. + string sandboxer = 10; + } diff --git a/client.go b/client.go index a369ab96c..b15f4e20c 100644 --- a/client.go +++ b/client.go @@ -719,9 +719,10 @@ func (c *Client) SandboxStore() sandbox.Store { } // SandboxController returns the underlying sandbox controller client -func (c *Client) SandboxController() sandbox.Controller { - if c.sandboxController != nil { - return c.sandboxController +func (c *Client) SandboxController(name string) sandbox.Controller { + // default sandboxer is shim + if c.sandboxers != nil { + return c.sandboxers[name] } c.connMu.Lock() defer c.connMu.Unlock() diff --git a/metadata/buckets.go b/metadata/buckets.go index e5254f0a6..9ccfaee83 100644 --- a/metadata/buckets.go +++ b/metadata/buckets.go @@ -114,6 +114,7 @@ │ ├──createdat : - Created at │ ├──updatedat : - Updated at │ ├──spec : - Proto marshaled spec + │ ├──sandboxer : - Sandboxer name │ ├──runtime │ │ ├──name : - Runtime name │ │ └──options : - Proto marshaled options @@ -173,6 +174,7 @@ var ( bucketKeyRef = []byte("ref") bucketKeyExpireAt = []byte("expireat") bucketKeySandboxID = []byte("sandboxid") + bucketKeySandboxer = []byte("sandboxer") deprecatedBucketKeyObjectIngest = []byte("ingest") // stores ingest links, deprecated in v1.2 ) diff --git a/metadata/sandbox.go b/metadata/sandbox.go index 81bee2b75..542a1b3c0 100644 --- a/metadata/sandbox.go +++ b/metadata/sandbox.go @@ -292,6 +292,10 @@ func (s *sandboxStore) write(parent *bbolt.Bucket, instance *api.Sandbox, overwr return err } + if err := bucket.Put(bucketKeySandboxer, []byte(instance.Sandboxer)); err != nil { + return err + } + runtimeBucket, err := bucket.CreateBucketIfNotExists(bucketKeyRuntime) if err != nil { return err @@ -350,6 +354,12 @@ func (s *sandboxStore) read(parent *bbolt.Bucket, id []byte) (api.Sandbox, error if err != nil { return api.Sandbox{}, err } + sandboxer := bucket.Get(bucketKeySandboxer) + if sandboxer == nil { + inst.Sandboxer = "" + } else { + inst.Sandboxer = string(sandboxer) + } return inst, nil } diff --git a/metadata/sandbox_test.go b/metadata/sandbox_test.go index 711fb0c13..848708908 100644 --- a/metadata/sandbox_test.go +++ b/metadata/sandbox_test.go @@ -43,6 +43,7 @@ func TestSandboxCreate(t *testing.T) { Name: "test", Options: &types.Any{TypeUrl: "url/3", Value: []byte{4, 5, 6}}, }, + Sandboxer: "test-sandboxer", } _, err := store.Create(ctx, in) @@ -91,7 +92,8 @@ func TestSandboxUpdate(t *testing.T) { Extensions: map[string]typeurl.Any{ "ext2": &types.Any{TypeUrl: "url2", Value: []byte{4, 5, 6}}, // will append `ext1` }, - Runtime: api.RuntimeOpts{Name: "test"}, // no change + Runtime: api.RuntimeOpts{Name: "test"}, // no change + Sandboxer: "test-sandboxer", // no change }); err != nil { t.Fatal(err) } @@ -120,7 +122,8 @@ func TestSandboxUpdate(t *testing.T) { "ext1": &types.Any{TypeUrl: "url1", Value: []byte{1, 2}}, "ext2": &types.Any{TypeUrl: "url2", Value: []byte{4, 5, 6}}, }, - Runtime: api.RuntimeOpts{Name: "test"}, + Runtime: api.RuntimeOpts{Name: "test"}, + Sandboxer: "test-sandboxer", } assertEqualInstances(t, out, expected) @@ -158,7 +161,8 @@ func TestSandboxList(t *testing.T) { TypeUrl: "test", Value: []byte{9}, }}, - Runtime: api.RuntimeOpts{Name: "test"}, + Runtime: api.RuntimeOpts{Name: "test"}, + Sandboxer: "test-sandboxer", }, } @@ -203,7 +207,8 @@ func TestSandboxListWithFilter(t *testing.T) { TypeUrl: "test", Value: []byte{9}, }}, - Runtime: api.RuntimeOpts{Name: "test"}, + Runtime: api.RuntimeOpts{Name: "test"}, + Sandboxer: "test-sandboxer", }, } diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 279a2a56d..038ad19f5 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -74,11 +74,11 @@ type Runtime struct { // while using default snapshotters for operational simplicity. // See https://github.com/containerd/containerd/issues/6657 for details. Snapshotter string `toml:"snapshotter" json:"snapshotter"` - // SandboxMode defines which sandbox runtime to use when scheduling pods + // Sandboxer defines which sandbox runtime to use when scheduling pods // This features requires the new CRI server implementation (enabled by default in 2.0) // shim - means use whatever Controller implementation provided by shim (e.g. use RemoteController). // podsandbox - means use Controller implementation from sbserver podsandbox package. - SandboxMode string `toml:"sandbox_mode" json:"sandboxMode"` + Sandboxer string `toml:"sandboxer" json:"sandboxer"` } // ContainerdConfig contains toml config related to containerd @@ -383,8 +383,8 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) error { return errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled") } // If empty, use default podSandbox mode - if len(r.SandboxMode) == 0 { - r.SandboxMode = string(ModePodSandbox) + if len(r.Sandboxer) == 0 { + r.Sandboxer = string(ModePodSandbox) c.ContainerdConfig.Runtimes[k] = r } } diff --git a/pkg/cri/config/config_test.go b/pkg/cri/config/config_test.go index 171228e99..5c021857e 100644 --- a/pkg/cri/config/config_test.go +++ b/pkg/cri/config/config_test.go @@ -61,7 +61,7 @@ func TestValidateConfig(t *testing.T) { DefaultRuntimeName: RuntimeDefault, Runtimes: map[string]Runtime{ RuntimeDefault: { - SandboxMode: string(ModePodSandbox), + Sandboxer: string(ModePodSandbox), }, }, }, diff --git a/pkg/cri/config/config_unix.go b/pkg/cri/config/config_unix.go index 29476ee4c..76bbe9148 100644 --- a/pkg/cri/config/config_unix.go +++ b/pkg/cri/config/config_unix.go @@ -69,9 +69,9 @@ func DefaultConfig() PluginConfig { DefaultRuntimeName: "runc", Runtimes: map[string]Runtime{ "runc": { - Type: "io.containerd.runc.v2", - Options: m, - SandboxMode: string(ModePodSandbox), + Type: "io.containerd.runc.v2", + Options: m, + Sandboxer: string(ModePodSandbox), }, }, DisableSnapshotAnnotations: true, diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index b1bf08960..93c584c93 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -274,10 +274,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta containerd.WithContainerExtension(containerMetadataExtension, &meta), ) - // When using sandboxed shims, containerd's runtime needs to know which sandbox shim instance to use. - if ociRuntime.SandboxMode == string(criconfig.ModeShim) { - opts = append(opts, containerd.WithSandbox(sandboxID)) - } + opts = append(opts, containerd.WithSandbox(sandboxID)) opts = append(opts, c.nri.WithContainerAdjustment()) defer func() { diff --git a/pkg/cri/server/podsandbox/controller.go b/pkg/cri/server/podsandbox/controller.go index 78944616c..13ab1f9ab 100644 --- a/pkg/cri/server/podsandbox/controller.go +++ b/pkg/cri/server/podsandbox/controller.go @@ -34,10 +34,26 @@ import ( ctrdutil "github.com/containerd/containerd/pkg/cri/util" osinterface "github.com/containerd/containerd/pkg/os" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/plugin/registry" + "github.com/containerd/containerd/plugins" "github.com/containerd/containerd/protobuf" "github.com/containerd/containerd/sandbox" ) +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.SandboxControllerPlugin, + ID: "podsandbox", + Requires: []plugin.Type{}, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + // register the global controller to containerd plugin manager, + // the global controller will be initialized when cri plugin is initializing + return &Controller{}, nil + }, + }) +} + // CRIService interface contains things required by controller, but not yet refactored from criService. // TODO: this will be removed in subsequent iterations. type CRIService interface { @@ -72,7 +88,7 @@ type Controller struct { store *Store } -func New( +func (c *Controller) Init( config criconfig.Config, client *containerd.Client, sandboxStore *sandboxstore.Store, @@ -80,17 +96,15 @@ func New( cri CRIService, imageService ImageService, baseOCISpecs map[string]*oci.Spec, -) *Controller { - return &Controller{ - config: config, - client: client, - imageService: imageService, - sandboxStore: sandboxStore, - os: os, - cri: cri, - baseOCISpecs: baseOCISpecs, - store: NewStore(), - } +) { + c.cri = cri + c.client = client + c.config = config + c.sandboxStore = sandboxStore + c.os = os + c.baseOCISpecs = baseOCISpecs + c.store = NewStore() + c.imageService = imageService } var _ sandbox.Controller = (*Controller)(nil) diff --git a/pkg/cri/server/podsandbox/sandbox_run.go b/pkg/cri/server/podsandbox/sandbox_run.go index fc5cbd2c3..07eee22f5 100644 --- a/pkg/cri/server/podsandbox/sandbox_run.go +++ b/pkg/cri/server/podsandbox/sandbox_run.go @@ -273,7 +273,7 @@ func (c *Controller) Start(ctx context.Context, id string) (cin sandbox.Controll return } -func (c *Controller) Create(ctx context.Context, _id string, _ ...sandbox.CreateOpt) error { +func (c *Controller) Create(ctx context.Context, _info sandbox.Sandbox, _ ...sandbox.CreateOpt) error { // Not used by pod-sandbox implementation as there is no need to split pause containers logic. return nil } diff --git a/pkg/cri/server/restart.go b/pkg/cri/server/restart.go index 8b7455c4d..027878162 100644 --- a/pkg/cri/server/restart.go +++ b/pkg/cri/server/restart.go @@ -60,10 +60,7 @@ func (c *criService) recover(ctx context.Context) error { return fmt.Errorf("failed to list sandbox containers: %w", err) } - podSandboxController, ok := c.sandboxControllers[criconfig.ModePodSandbox] - if !ok { - log.G(ctx).Fatal("unable to restore pod sandboxes, no controller found") - } + podSandboxController := c.client.SandboxController(string(criconfig.ModePodSandbox)) podSandboxLoader, ok := podSandboxController.(podSandboxRecover) if !ok { @@ -115,7 +112,7 @@ func (c *criService) recover(ctx context.Context) error { var ( state = sandboxstore.StateUnknown - controller = c.sandboxControllers[criconfig.ModeShim] + controller = c.client.SandboxController(sbx.Sandboxer) ) status, err := controller.Status(ctx, sbx.ID, false) diff --git a/pkg/cri/server/sandbox_run.go b/pkg/cri/server/sandbox_run.go index 6dbecdc38..4e553ed1a 100644 --- a/pkg/cri/server/sandbox_run.go +++ b/pkg/cri/server/sandbox_run.go @@ -92,6 +92,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox } sandboxInfo.Runtime.Name = ociRuntime.Type + sandboxInfo.Sandboxer = ociRuntime.Sandboxer runtimeStart := time.Now() // Retrieve runtime options @@ -248,7 +249,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("unable to update extensions for sandbox %q: %w", id, err) } - if err := controller.Create(ctx, id, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil { + if err := controller.Create(ctx, sandboxInfo, sb.WithOptions(config), sb.WithNetNSPath(sandbox.NetNSPath)); err != nil { return nil, fmt.Errorf("failed to create sandbox %q: %w", id, err) } @@ -352,7 +353,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox } // TODO: get rid of this. sandbox object should no longer have Container field. - if ociRuntime.SandboxMode == string(criconfig.ModePodSandbox) { + if ociRuntime.Sandboxer == string(criconfig.ModePodSandbox) { container, err := c.client.LoadContainer(ctx, id) if err != nil { return nil, fmt.Errorf("failed to load container %q for sandbox: %w", id, err) @@ -683,25 +684,6 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtime return handler, nil } -// getSandboxController returns the sandbox controller configuration for sandbox. -// If absent in legacy case, it will return the default controller. -func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sb.Controller, error) { - ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler) - if err != nil { - return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) - } - // Validate mode - if err = ValidateMode(ociRuntime.SandboxMode); err != nil { - return nil, err - } - // Use sandbox controller to delete sandbox - controller, exist := c.sandboxControllers[criconfig.SandboxControllerMode(ociRuntime.SandboxMode)] - if !exist { - return nil, fmt.Errorf("sandbox controller %s not exist", ociRuntime.SandboxMode) - } - return controller, nil -} - func logDebugCNIResult(ctx context.Context, sandboxID string, result *cni.Result) { if log.GetLevel() < log.DebugLevel { return diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 3af3af79c..cfec64cd6 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -98,9 +98,6 @@ type criService struct { sandboxNameIndex *registrar.Registrar // containerStore stores all resources associated with containers. containerStore *containerstore.Store - // sandboxControllers contains different sandbox controller type, - // every controller controls sandbox lifecycle (and hides implementation details behind). - sandboxControllers map[criconfig.SandboxControllerMode]sandbox.Controller // containerNameIndex stores all container names and make sure each // name is unique. containerNameIndex *registrar.Registrar @@ -169,7 +166,6 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. sandboxNameIndex: registrar.NewRegistrar(), containerNameIndex: registrar.NewRegistrar(), netPlugin: make(map[string]cni.CNI), - sandboxControllers: make(map[criconfig.SandboxControllerMode]sandbox.Controller), } // TODO: figure out a proper channel size. @@ -210,9 +206,8 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, err } - // Load all sandbox controllers(pod sandbox controller and remote shim controller) - c.sandboxControllers[criconfig.ModePodSandbox] = podsandbox.New(config, client, c.sandboxStore, c.os, c, imageService, c.baseOCISpecs) - c.sandboxControllers[criconfig.ModeShim] = client.SandboxController() + podSandboxController := c.client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller) + podSandboxController.Init(config, client, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) c.nri = nri @@ -357,6 +352,17 @@ func (c *criService) register(s *grpc.Server) error { return nil } +// getSandboxController returns the sandbox controller configuration for sandbox. +// If absent in legacy case, it will return the default controller. +func (c *criService) getSandboxController(config *runtime.PodSandboxConfig, runtimeHandler string) (sandbox.Controller, error) { + ociRuntime, err := c.getSandboxRuntime(config, runtimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox runtime: %w", err) + } + + return c.client.SandboxController(ociRuntime.Sandboxer), nil +} + // imageFSPath returns containerd image filesystem path. // Note that if containerd changes directory layout, we also needs to change this. func imageFSPath(rootDir, snapshotter string) string { diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 12769e266..39db099bf 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -42,7 +42,7 @@ import ( func init() { registry.Register(&plugin.Registration{ Type: plugins.SandboxControllerPlugin, - ID: "local", + ID: "shim", Requires: []plugin.Type{ plugins.RuntimePluginV2, plugins.EventPlugin, @@ -59,20 +59,13 @@ func init() { return nil, err } - sbPlugin, err := ic.GetByID(plugins.SandboxStorePlugin, "local") - if err != nil { - return nil, err - } - var ( shims = shimPlugin.(*v2.ShimManager) publisher = exchangePlugin.(*exchange.Exchange) - store = sbPlugin.(sandbox.Store) ) return &controllerLocal{ shims: shims, - store: store, publisher: publisher, }, nil }, @@ -81,7 +74,6 @@ func init() { type controllerLocal struct { shims *v2.ShimManager - store sandbox.Store publisher events.Publisher } @@ -106,8 +98,9 @@ func (c *controllerLocal) cleanupShim(ctx context.Context, sandboxID string, svc } } -func (c *controllerLocal) Create(ctx context.Context, sandboxID string, opts ...sandbox.CreateOpt) error { +func (c *controllerLocal) Create(ctx context.Context, info sandbox.Sandbox, opts ...sandbox.CreateOpt) error { var coptions sandbox.CreateOptions + sandboxID := info.ID for _, opt := range opts { opt(&coptions) } @@ -116,11 +109,6 @@ func (c *controllerLocal) Create(ctx context.Context, sandboxID string, opts ... return fmt.Errorf("sandbox %s already running: %w", sandboxID, errdefs.ErrAlreadyExists) } - info, err := c.store.Get(ctx, sandboxID) - if err != nil { - return fmt.Errorf("failed to query sandbox metadata from store: %w", err) - } - shim, err := c.shims.Start(ctx, sandboxID, runtime.CreateOpts{ Spec: info.Spec, RuntimeOptions: info.Runtime.Options, diff --git a/runtime/v2/shim.go b/runtime/v2/shim.go index 46452b55b..4b6172602 100644 --- a/runtime/v2/shim.go +++ b/runtime/v2/shim.go @@ -129,19 +129,6 @@ func loadShim(ctx context.Context, bundle *Bundle, onClose func()) (_ ShimInstan client: conn, } - ctx, cancel := timeout.WithContext(ctx, loadTimeout) - defer cancel() - - // Check connectivity, TaskService is the only required service, so create a temp one to check connection. - s, err := newShimTask(shim) - if err != nil { - return nil, err - } - - if _, err := s.PID(ctx); err != nil { - return nil, err - } - return shim, nil } diff --git a/runtime/v2/shim_load.go b/runtime/v2/shim_load.go index 7b0b4c8b7..5556e6c4d 100644 --- a/runtime/v2/shim_load.go +++ b/runtime/v2/shim_load.go @@ -26,6 +26,7 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/pkg/cleanup" + "github.com/containerd/containerd/pkg/timeout" "github.com/containerd/log" ) @@ -141,7 +142,7 @@ func (m *ShimManager) loadShims(ctx context.Context) error { ttrpcAddress: m.containerdTTRPCAddress, schedCore: m.schedCore, }) - instance, err := loadShim(ctx, bundle, func() { + shim, err := loadShimTask(ctx, bundle, func() { log.G(ctx).WithField("id", id).Info("shim disconnected") cleanupAfterDeadShim(cleanup.Background(ctx), id, m.shims, m.events, binaryCall) @@ -152,10 +153,6 @@ func (m *ShimManager) loadShims(ctx context.Context) error { cleanupAfterDeadShim(ctx, id, m.shims, m.events, binaryCall) continue } - shim, err := newShimTask(instance) - if err != nil { - return err - } // There are 3 possibilities for the loaded shim here: // 1. It could be a shim that is running a task. @@ -180,6 +177,26 @@ func (m *ShimManager) loadShims(ctx context.Context) error { return nil } +func loadShimTask(ctx context.Context, bundle *Bundle, onClose func()) (_ *shimTask, retErr error) { + shim, err := loadShim(ctx, bundle, onClose) + if err != nil { + return nil, err + } + // Check connectivity, TaskService is the only required service, so create a temp one to check connection. + s, err := newShimTask(shim) + if err != nil { + return nil, err + } + + ctx, cancel := timeout.WithContext(ctx, loadTimeout) + defer cancel() + + if _, err := s.PID(ctx); err != nil { + return nil, err + } + return s, nil +} + func (m *ShimManager) cleanupWorkDirs(ctx context.Context) error { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { diff --git a/sandbox.go b/sandbox.go index 361f984ff..d9041c84e 100644 --- a/sandbox.go +++ b/sandbox.go @@ -23,6 +23,7 @@ import ( "time" "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/oci" "github.com/containerd/containerd/protobuf/types" api "github.com/containerd/containerd/sandbox" @@ -81,7 +82,7 @@ func (s *sandboxClient) Labels(ctx context.Context) (map[string]string, error) { } func (s *sandboxClient) Start(ctx context.Context) error { - resp, err := s.client.SandboxController().Start(ctx, s.ID()) + resp, err := s.client.SandboxController(s.metadata.Sandboxer).Start(ctx, s.ID()) if err != nil { return err } @@ -95,7 +96,7 @@ func (s *sandboxClient) Wait(ctx context.Context) (<-chan ExitStatus, error) { go func() { defer close(c) - exitStatus, err := s.client.SandboxController().Wait(ctx, s.ID()) + exitStatus, err := s.client.SandboxController(s.metadata.Sandboxer).Wait(ctx, s.ID()) if err != nil { c <- ExitStatus{ code: UnknownExitStatus, @@ -114,15 +115,15 @@ func (s *sandboxClient) Wait(ctx context.Context) (<-chan ExitStatus, error) { } func (s *sandboxClient) Stop(ctx context.Context) error { - return s.client.SandboxController().Stop(ctx, s.ID()) + return s.client.SandboxController(s.metadata.Sandboxer).Stop(ctx, s.ID()) } func (s *sandboxClient) Shutdown(ctx context.Context) error { - if err := s.client.SandboxController().Shutdown(ctx, s.ID()); err != nil { + if err := s.client.SandboxController(s.metadata.Sandboxer).Shutdown(ctx, s.ID()); err != nil && errdefs.IsNotFound(err) { return fmt.Errorf("failed to shutdown sandbox: %w", err) } - if err := s.client.SandboxStore().Delete(ctx, s.ID()); err != nil { + if err := s.client.SandboxStore().Delete(ctx, s.ID()); err != nil && !errdefs.IsNotFound(err) { return fmt.Errorf("failed to delete sandbox from store: %w", err) } @@ -166,7 +167,7 @@ func (c *Client) LoadSandbox(ctx context.Context, id string) (Sandbox, error) { return nil, err } - status, err := c.SandboxController().Status(ctx, id, false) + status, err := c.SandboxController(sandbox.Sandboxer).Status(ctx, id, false) if err != nil { return nil, fmt.Errorf("failed to load sandbox %s, status request failed: %w", id, err) } diff --git a/sandbox/controller.go b/sandbox/controller.go index 45264e791..1beacdde4 100644 --- a/sandbox/controller.go +++ b/sandbox/controller.go @@ -94,7 +94,7 @@ func WithTimeout(timeout time.Duration) StopOpt { // Shim lifetimes are now managed manually via sandbox API by the containerd's client. type Controller interface { // Create is used to initialize sandbox environment. (mounts, any) - Create(ctx context.Context, sandboxID string, opts ...CreateOpt) error + Create(ctx context.Context, sandboxInfo Sandbox, opts ...CreateOpt) error // Start will start previously created sandbox. Start(ctx context.Context, sandboxID string) (ControllerInstance, error) // Platform returns target sandbox OS that will be used by Controller. diff --git a/sandbox/helpers.go b/sandbox/helpers.go index bfe0b23d3..09667adb6 100644 --- a/sandbox/helpers.go +++ b/sandbox/helpers.go @@ -35,6 +35,7 @@ func ToProto(sandbox *Sandbox) *types.Sandbox { Name: sandbox.Runtime.Name, Options: protobuf.FromAny(sandbox.Runtime.Options), }, + Sandboxer: sandbox.Sandboxer, Labels: sandbox.Labels, CreatedAt: protobuf.ToTimestamp(sandbox.CreatedAt), UpdatedAt: protobuf.ToTimestamp(sandbox.UpdatedAt), @@ -61,6 +62,7 @@ func FromProto(sandboxpb *types.Sandbox) Sandbox { Labels: sandboxpb.Labels, Runtime: runtime, Spec: sandboxpb.Spec, + Sandboxer: sandboxpb.Sandboxer, CreatedAt: protobuf.FromTimestamp(sandboxpb.CreatedAt), UpdatedAt: protobuf.FromTimestamp(sandboxpb.UpdatedAt), Extensions: extensions, diff --git a/sandbox/proxy/controller.go b/sandbox/proxy/controller.go index 770b561b7..c9f59da67 100644 --- a/sandbox/proxy/controller.go +++ b/sandbox/proxy/controller.go @@ -40,13 +40,13 @@ func NewSandboxController(client api.ControllerClient) sandbox.Controller { return &remoteSandboxController{client: client} } -func (s *remoteSandboxController) Create(ctx context.Context, sandboxID string, opts ...sandbox.CreateOpt) error { +func (s *remoteSandboxController) Create(ctx context.Context, sandboxInfo sandbox.Sandbox, opts ...sandbox.CreateOpt) error { var options sandbox.CreateOptions for _, opt := range opts { opt(&options) } _, err := s.client.Create(ctx, &api.ControllerCreateRequest{ - SandboxID: sandboxID, + SandboxID: sandboxInfo.ID, Rootfs: mount.ToProto(options.Rootfs), Options: &anypb.Any{ TypeUrl: options.Options.GetTypeUrl(), diff --git a/sandbox/store.go b/sandbox/store.go index cda646dde..6cffafb9b 100644 --- a/sandbox/store.go +++ b/sandbox/store.go @@ -35,6 +35,8 @@ type Sandbox struct { Runtime RuntimeOpts // Spec carries the runtime specification used to implement the sandbox Spec typeurl.Any + // Sandboxer is the sandbox controller who manages the sandbox + Sandboxer string // CreatedAt is the time at which the sandbox was created CreatedAt time.Time // UpdatedAt is the time at which the sandbox was updated diff --git a/services.go b/services.go index 410f57921..c9c9ba544 100644 --- a/services.go +++ b/services.go @@ -50,7 +50,7 @@ type services struct { leasesService leases.Manager introspectionService introspection.Service sandboxStore sandbox.Store - sandboxController sandbox.Controller + sandboxers map[string]sandbox.Controller } // ServicesOpt allows callers to set options on the services @@ -87,6 +87,16 @@ func WithSnapshotters(snapshotters map[string]snapshots.Snapshotter) ServicesOpt } } +// WithSandboxers sets the sandbox controllers. +func WithSandboxers(sandboxers map[string]sandbox.Controller) ServicesOpt { + return func(s *services) { + s.sandboxers = make(map[string]sandbox.Controller) + for n, sn := range sandboxers { + s.sandboxers[n] = sn + } + } +} + // WithContainerClient sets the container service to use using a containers client. func WithContainerClient(containerService containersapi.ContainersClient) ServicesOpt { return func(s *services) { @@ -171,13 +181,6 @@ func WithSandboxStore(client sandbox.Store) ServicesOpt { } } -// WithSandboxController sets the sandbox controller. -func WithSandboxController(client sandbox.Controller) ServicesOpt { - return func(s *services) { - s.sandboxController = client - } -} - // WithInMemoryServices is suitable for cases when there is need to use containerd's client from // another (in-memory) containerd plugin (such as CRI). func WithInMemoryServices(ic *plugin.InitContext) ClientOpt { @@ -193,9 +196,6 @@ func WithInMemoryServices(ic *plugin.InitContext) ClientOpt { plugins.SandboxStorePlugin: func(i interface{}) ServicesOpt { return WithSandboxStore(i.(sandbox.Store)) }, - plugins.SandboxControllerPlugin: func(i interface{}) ServicesOpt { - return WithSandboxController(i.(sandbox.Controller)) - }, } { i, err := ic.Get(t) if err != nil { @@ -218,6 +218,9 @@ func WithInMemoryServices(ic *plugin.InitContext) ClientOpt { srv.SnapshotsService: func(s interface{}) ServicesOpt { return WithSnapshotters(s.(map[string]snapshots.Snapshotter)) }, + srv.SandboxControllersService: func(s interface{}) ServicesOpt { + return WithSandboxers(s.(map[string]sandbox.Controller)) + }, srv.ContainersService: func(s interface{}) ServicesOpt { return WithContainerClient(s.(containersapi.ContainersClient)) }, diff --git a/services/sandbox/controller_service.go b/services/sandbox/controller_service.go index 487828bbc..2cb066fdb 100644 --- a/services/sandbox/controller_service.go +++ b/services/sandbox/controller_service.go @@ -18,6 +18,9 @@ package sandbox import ( "context" + "errors" + "fmt" + "time" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/anypb" @@ -31,6 +34,7 @@ import ( "github.com/containerd/containerd/plugins" "github.com/containerd/containerd/protobuf" "github.com/containerd/containerd/sandbox" + "github.com/containerd/containerd/services" "github.com/containerd/log" ) @@ -39,22 +43,30 @@ func init() { Type: plugins.GRPCPlugin, ID: "sandbox-controllers", Requires: []plugin.Type{ - plugins.SandboxControllerPlugin, + plugins.ServicePlugin, plugins.EventPlugin, }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - sc, err := ic.GetByID(plugins.SandboxControllerPlugin, "local") + plugs, err := ic.GetByType(plugins.ServicePlugin) if err != nil { return nil, err } - + p, ok := plugs[services.SandboxControllersService] + if !ok { + return nil, errors.New("sandboxes service not found") + } + i, err := p.Instance() + if err != nil { + return nil, err + } + sc := i.(map[string]sandbox.Controller) ep, err := ic.Get(plugins.EventPlugin) if err != nil { return nil, err } return &controllerService{ - local: sc.(sandbox.Controller), + sc: sc, publisher: ep.(events.Publisher), }, nil }, @@ -62,7 +74,7 @@ func init() { } type controllerService struct { - local sandbox.Controller + sc map[string]sandbox.Controller publisher events.Publisher api.UnimplementedControllerServer } @@ -74,10 +86,24 @@ func (s *controllerService) Register(server *grpc.Server) error { return nil } +func (s *controllerService) getController(name string) (sandbox.Controller, error) { + if len(name) == 0 { + return nil, fmt.Errorf("%w: sandbox controller name can not be empty", errdefs.ErrInvalidArgument) + } + if ctrl, ok := s.sc[name]; ok { + return ctrl, nil + } + return nil, fmt.Errorf("%w: failed to get sandbox controller by %s", errdefs.ErrNotFound, name) +} + func (s *controllerService) Create(ctx context.Context, req *api.ControllerCreateRequest) (*api.ControllerCreateResponse, error) { log.G(ctx).WithField("req", req).Debug("create sandbox") // TODO: Rootfs - err := s.local.Create(ctx, req.GetSandboxID(), sandbox.WithOptions(req.GetOptions())) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + err = ctrl.Create(ctx, sandbox.Sandbox{ID: req.GetSandboxID()}, sandbox.WithOptions(req.GetOptions())) if err != nil { return &api.ControllerCreateResponse{}, errdefs.ToGRPC(err) } @@ -95,7 +121,11 @@ func (s *controllerService) Create(ctx context.Context, req *api.ControllerCreat func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartRequest) (*api.ControllerStartResponse, error) { log.G(ctx).WithField("req", req).Debug("start sandbox") - inst, err := s.local.Start(ctx, req.GetSandboxID()) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + inst, err := ctrl.Start(ctx, req.GetSandboxID()) if err != nil { return &api.ControllerStartResponse{}, errdefs.ToGRPC(err) } @@ -116,12 +146,20 @@ func (s *controllerService) Start(ctx context.Context, req *api.ControllerStartR func (s *controllerService) Stop(ctx context.Context, req *api.ControllerStopRequest) (*api.ControllerStopResponse, error) { log.G(ctx).WithField("req", req).Debug("delete sandbox") - return &api.ControllerStopResponse{}, errdefs.ToGRPC(s.local.Stop(ctx, req.GetSandboxID())) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + return &api.ControllerStopResponse{}, errdefs.ToGRPC(ctrl.Stop(ctx, req.GetSandboxID(), sandbox.WithTimeout(time.Duration(req.TimeoutSecs)*time.Second))) } func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitRequest) (*api.ControllerWaitResponse, error) { log.G(ctx).WithField("req", req).Debug("wait sandbox") - exitStatus, err := s.local.Wait(ctx, req.GetSandboxID()) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + exitStatus, err := ctrl.Wait(ctx, req.GetSandboxID()) if err != nil { return &api.ControllerWaitResponse{}, errdefs.ToGRPC(err) } @@ -142,7 +180,11 @@ func (s *controllerService) Wait(ctx context.Context, req *api.ControllerWaitReq func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatusRequest) (*api.ControllerStatusResponse, error) { log.G(ctx).WithField("req", req).Debug("sandbox status") - cstatus, err := s.local.Status(ctx, req.GetSandboxID(), req.GetVerbose()) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + cstatus, err := ctrl.Status(ctx, req.GetSandboxID(), req.GetVerbose()) if err != nil { return &api.ControllerStatusResponse{}, errdefs.ToGRPC(err) } @@ -166,13 +208,20 @@ func (s *controllerService) Status(ctx context.Context, req *api.ControllerStatu func (s *controllerService) Shutdown(ctx context.Context, req *api.ControllerShutdownRequest) (*api.ControllerShutdownResponse, error) { log.G(ctx).WithField("req", req).Debug("shutdown sandbox") - return &api.ControllerShutdownResponse{}, errdefs.ToGRPC(s.local.Shutdown(ctx, req.GetSandboxID())) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + return &api.ControllerShutdownResponse{}, errdefs.ToGRPC(ctrl.Shutdown(ctx, req.GetSandboxID())) } func (s *controllerService) Metrics(ctx context.Context, req *api.ControllerMetricsRequest) (*api.ControllerMetricsResponse, error) { log.G(ctx).WithField("req", req).Debug("sandbox metrics") - - metrics, err := s.local.Metrics(ctx, req.GetSandboxID()) + ctrl, err := s.getController(req.Sandboxer) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + metrics, err := ctrl.Metrics(ctx, req.GetSandboxID()) if err != nil { return &api.ControllerMetricsResponse{}, errdefs.ToGRPC(err) } diff --git a/services/sandbox/sandboxers.go b/services/sandbox/sandboxers.go new file mode 100644 index 000000000..6d9ab4c9c --- /dev/null +++ b/services/sandbox/sandboxers.go @@ -0,0 +1,50 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sandbox + +import ( + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/plugin/registry" + "github.com/containerd/containerd/plugins" + "github.com/containerd/containerd/sandbox" + "github.com/containerd/containerd/services" +) + +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.ServicePlugin, + ID: services.SandboxControllersService, + Requires: []plugin.Type{ + plugins.SandboxControllerPlugin, + }, + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + sandboxesRaw, err := ic.GetByType(plugins.SandboxControllerPlugin) + if err != nil { + return nil, err + } + sandboxers := make(map[string]sandbox.Controller) + for name, srv := range sandboxesRaw { + inst, err := srv.Instance() + if err != nil { + return nil, err + } + sandboxers[name] = inst.(sandbox.Controller) + } + return sandboxers, nil + }, + }) +} diff --git a/services/server/server.go b/services/server/server.go index 5e63f3902..864deae56 100644 --- a/services/server/server.go +++ b/services/server/server.go @@ -36,6 +36,7 @@ import ( csapi "github.com/containerd/containerd/api/services/content/v1" diffapi "github.com/containerd/containerd/api/services/diff/v1" + sbapi "github.com/containerd/containerd/api/services/sandbox/v1" ssapi "github.com/containerd/containerd/api/services/snapshots/v1" "github.com/containerd/containerd/content/local" csproxy "github.com/containerd/containerd/content/proxy" @@ -49,6 +50,7 @@ import ( "github.com/containerd/containerd/plugin/dynamic" "github.com/containerd/containerd/plugin/registry" "github.com/containerd/containerd/plugins" + sbproxy "github.com/containerd/containerd/sandbox/proxy" srvconfig "github.com/containerd/containerd/services/server/config" ssproxy "github.com/containerd/containerd/snapshots/proxy" "github.com/containerd/containerd/sys" @@ -474,6 +476,11 @@ func LoadPlugins(ctx context.Context, config *srvconfig.Config) ([]plugin.Regist f = func(conn *grpc.ClientConn) interface{} { return csproxy.NewContentStore(csapi.NewContentClient(conn)) } + case string(plugins.SandboxControllerPlugin), "sandbox": + t = plugins.SandboxControllerPlugin + f = func(conn *grpc.ClientConn) interface{} { + return sbproxy.NewSandboxController(sbapi.NewControllerClient(conn)) + } case string(plugins.DiffPlugin), "diff": t = plugins.DiffPlugin f = func(conn *grpc.ClientConn) interface{} { diff --git a/services/services.go b/services/services.go index a058603b6..7c794de2a 100644 --- a/services/services.go +++ b/services/services.go @@ -21,6 +21,8 @@ const ( ContentService = "content-service" // SnapshotsService is id of snapshots service. SnapshotsService = "snapshots-service" + // SandboxControllersService is id of snapshots service. + SandboxControllersService = "sandboxes-service" // ImagesService is id of images service. ImagesService = "images-service" // ContainersService is id of containers service.