From f6e0cf1894b9211f07d801e1af3166f290969cd7 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Thu, 1 Feb 2024 20:18:54 +0800 Subject: [PATCH 1/5] sandbox: add address info in Start and Status response Signed-off-by: Abel Feng --- api/next.pb.txt | 42 +++++++++++++++++ api/services/sandbox/v1/sandbox.pb.go | 66 +++++++++++++++++++++++++-- api/services/sandbox/v1/sandbox.proto | 6 +++ 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/api/next.pb.txt b/api/next.pb.txt index 5f3f591bc..aa77f2f00 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -5506,6 +5506,27 @@ file { type_name: ".containerd.services.sandbox.v1.ControllerStartResponse.LabelsEntry" json_name: "labels" } + field { + name: "address" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "address" + } + field { + name: "version" + number: 6 + label: LABEL_OPTIONAL + type: TYPE_UINT32 + json_name: "version" + } + field { + name: "protocol" + number: 7 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "protocol" + } nested_type { name: "LabelsEntry" field { @@ -5696,6 +5717,27 @@ file { type_name: ".google.protobuf.Any" json_name: "extra" } + field { + name: "address" + number: 8 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "address" + } + field { + name: "version" + number: 9 + label: LABEL_OPTIONAL + type: TYPE_UINT32 + json_name: "version" + } + field { + name: "protocol" + number: 10 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "protocol" + } nested_type { name: "InfoEntry" field { diff --git a/api/services/sandbox/v1/sandbox.pb.go b/api/services/sandbox/v1/sandbox.pb.go index 64af04b9c..52e20826a 100644 --- a/api/services/sandbox/v1/sandbox.pb.go +++ b/api/services/sandbox/v1/sandbox.pb.go @@ -713,6 +713,9 @@ type ControllerStartResponse struct { Pid uint32 `protobuf:"varint,2,opt,name=pid,proto3" json:"pid,omitempty"` CreatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` Labels map[string]string `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Address string `protobuf:"bytes,5,opt,name=address,proto3" json:"address,omitempty"` + Version uint32 `protobuf:"varint,6,opt,name=version,proto3" json:"version,omitempty"` + Protocol string `protobuf:"bytes,7,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *ControllerStartResponse) Reset() { @@ -775,6 +778,27 @@ func (x *ControllerStartResponse) GetLabels() map[string]string { return nil } +func (x *ControllerStartResponse) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *ControllerStartResponse) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ControllerStartResponse) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + type ControllerPlatformRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1163,6 +1187,9 @@ type ControllerStatusResponse struct { CreatedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` ExitedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=exited_at,json=exitedAt,proto3" json:"exited_at,omitempty"` Extra *anypb.Any `protobuf:"bytes,7,opt,name=extra,proto3" json:"extra,omitempty"` + Address string `protobuf:"bytes,8,opt,name=address,proto3" json:"address,omitempty"` + Version uint32 `protobuf:"varint,9,opt,name=version,proto3" json:"version,omitempty"` + Protocol string `protobuf:"bytes,10,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *ControllerStatusResponse) Reset() { @@ -1246,6 +1273,27 @@ func (x *ControllerStatusResponse) GetExtra() *anypb.Any { return nil } +func (x *ControllerStatusResponse) GetAddress() string { + if x != nil { + return x.Address + } + return "" +} + +func (x *ControllerStatusResponse) GetVersion() uint32 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ControllerStatusResponse) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + type ControllerShutdownRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1542,7 +1590,7 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, - 0x72, 0x22, 0x9d, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x72, 0x22, 0xed, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, @@ -1556,7 +1604,12 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, @@ -1600,7 +1653,7 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, 0x65, 0x72, 0x62, 0x6f, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, - 0x22, 0x92, 0x03, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, + 0x22, 0xe2, 0x03, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, @@ -1621,7 +1674,12 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2a, 0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x1a, 0x37, 0x0a, 0x09, + 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x12, 0x18, 0x0a, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x1a, 0x37, 0x0a, 0x09, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, diff --git a/api/services/sandbox/v1/sandbox.proto b/api/services/sandbox/v1/sandbox.proto index fb57e775b..28de269d4 100755 --- a/api/services/sandbox/v1/sandbox.proto +++ b/api/services/sandbox/v1/sandbox.proto @@ -120,6 +120,9 @@ message ControllerStartResponse { uint32 pid = 2; google.protobuf.Timestamp created_at = 3; map labels = 4; + string address = 5; + uint32 version = 6; + string protocol = 7; } message ControllerPlatformRequest { @@ -163,6 +166,9 @@ message ControllerStatusResponse { google.protobuf.Timestamp created_at = 5; google.protobuf.Timestamp exited_at = 6; google.protobuf.Any extra = 7; + string address = 8; + uint32 version = 9; + string protocol = 10; } message ControllerShutdownRequest { From b1fefccc78a04078573f8d602e6e4be1eccda589 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Fri, 2 Feb 2024 10:59:31 +0800 Subject: [PATCH 2/5] sandbox: store endpoint in cri sandboxStore Signed-off-by: Abel Feng --- core/runtime/v2/shim.go | 20 ++++++++++++-------- core/sandbox/controller.go | 6 ++++++ core/sandbox/proxy/controller.go | 6 ++++++ internal/cri/server/container_start.go | 8 ++++++++ internal/cri/server/restart.go | 5 +++++ internal/cri/server/sandbox_run.go | 12 ++++++++++++ internal/cri/store/sandbox/sandbox.go | 12 ++++++++++++ plugins/sandbox/controller.go | 14 +++++++++++++- 8 files changed, 74 insertions(+), 9 deletions(-) diff --git a/core/runtime/v2/shim.go b/core/runtime/v2/shim.go index 0cdd18b27..89414bc43 100644 --- a/core/runtime/v2/shim.go +++ b/core/runtime/v2/shim.go @@ -185,8 +185,9 @@ type ShimInstance interface { Client() any // Delete will close the client and remove bundle from disk. Delete(ctx context.Context) error - // Version returns shim's features compatibility version. - Version() int + // Endpoint returns shim's endpoint information, + // including address, protocol and version. + Endpoint() (string, string, int) } func parseStartResponse(response []byte) (client.BootstrapParams, error) { @@ -359,9 +360,11 @@ func (gc *grpcConn) UserOnCloseWait(ctx context.Context) error { } type shim struct { - bundle *Bundle - client any - version int + bundle *Bundle + client any + address string + protocol string + version int } var _ ShimInstance = (*shim)(nil) @@ -371,8 +374,8 @@ func (s *shim) ID() string { return s.bundle.ID } -func (s *shim) Version() int { - return s.version +func (s *shim) Endpoint() (string, string, int) { + return s.address, s.protocol, s.version } func (s *shim) Namespace() string { @@ -440,7 +443,8 @@ type shimTask struct { } func newShimTask(shim ShimInstance) (*shimTask, error) { - taskClient, err := NewTaskClient(shim.Client(), shim.Version()) + _, _, version := shim.Endpoint() + taskClient, err := NewTaskClient(shim.Client(), version) if err != nil { return nil, err } diff --git a/core/sandbox/controller.go b/core/sandbox/controller.go index d30bb9f71..cf9a4302c 100644 --- a/core/sandbox/controller.go +++ b/core/sandbox/controller.go @@ -117,6 +117,9 @@ type ControllerInstance struct { SandboxID string Pid uint32 CreatedAt time.Time + Address string + Protocol string + Version uint32 Labels map[string]string } @@ -133,4 +136,7 @@ type ControllerStatus struct { CreatedAt time.Time ExitedAt time.Time Extra typeurl.Any + Address string + Protocol string + Version uint32 } diff --git a/core/sandbox/proxy/controller.go b/core/sandbox/proxy/controller.go index a4e5fddf1..661381367 100644 --- a/core/sandbox/proxy/controller.go +++ b/core/sandbox/proxy/controller.go @@ -73,6 +73,9 @@ func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) ( Pid: resp.GetPid(), CreatedAt: resp.GetCreatedAt().AsTime(), Labels: resp.GetLabels(), + Address: resp.GetAddress(), + Protocol: resp.GetProtocol(), + Version: resp.GetVersion(), }, nil } @@ -141,6 +144,9 @@ func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string, CreatedAt: resp.GetCreatedAt().AsTime(), ExitedAt: resp.GetExitedAt().AsTime(), Extra: resp.GetExtra(), + Address: resp.GetAddress(), + Protocol: resp.GetProtocol(), + Version: resp.GetVersion(), }, nil } diff --git a/internal/cri/server/container_start.go b/internal/cri/server/container_start.go index 549b6dffc..701339553 100644 --- a/internal/cri/server/container_start.go +++ b/internal/cri/server/container_start.go @@ -118,6 +118,14 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain if ociRuntime.Path != "" { taskOpts = append(taskOpts, containerd.WithRuntimePath(ociRuntime.Path)) } + + // append endpoint to the options so that task manager can get task api endpoint directly + endpoint := sandbox.Endpoint + if endpoint.IsValid() { + taskOpts = append(taskOpts, + containerd.WithTaskApiEndpoint(endpoint.Address, endpoint.Protocol, endpoint.Version)) + } + task, err := container.NewTask(ctx, ioCreation, taskOpts...) if err != nil { return nil, fmt.Errorf("failed to create containerd task: %w", err) diff --git a/internal/cri/server/restart.go b/internal/cri/server/restart.go index 3ac3d8032..e95698885 100644 --- a/internal/cri/server/restart.go +++ b/internal/cri/server/restart.go @@ -113,6 +113,7 @@ func (c *criService) recover(ctx context.Context) error { var ( state = sandboxstore.StateUnknown controller = c.client.SandboxController(sbx.Sandboxer) + endpoint sandboxstore.Endpoint ) status, err := controller.Status(ctx, sbx.ID, false) @@ -126,6 +127,9 @@ func (c *criService) recover(ctx context.Context) error { state = sandboxstore.StateNotReady } } else { + endpoint.Protocol = status.Protocol + endpoint.Version = status.Version + endpoint.Address = status.Address if code, ok := runtime.PodSandboxState_value[status.State]; ok { if code == int32(runtime.PodSandboxState_SANDBOX_READY) { state = sandboxstore.StateReady @@ -137,6 +141,7 @@ func (c *criService) recover(ctx context.Context) error { sb := sandboxstore.NewSandbox(metadata, sandboxstore.Status{State: state}) sb.Sandboxer = sbx.Sandboxer + sb.Endpoint = endpoint // Load network namespace. sb.NetNS = getNetNS(&metadata) diff --git a/internal/cri/server/sandbox_run.go b/internal/cri/server/sandbox_run.go index 7da4a30b9..4f5fda925 100644 --- a/internal/cri/server/sandbox_run.go +++ b/internal/cri/server/sandbox_run.go @@ -265,6 +265,18 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err) } + if ctrl.Protocol != "" && ctrl.Address != "" { + sandbox.Endpoint = sandboxstore.Endpoint{ + Protocol: ctrl.Protocol, + Version: ctrl.Version, + Address: ctrl.Address, + } + } + + if sandboxInfo, err = c.client.SandboxStore().Update(ctx, sandboxInfo, "extensions"); err != nil { + return nil, fmt.Errorf("unable to update extensions for sandbox %q: %w", id, err) + } + if !hostNetwork(config) && userNsEnabled { // If userns is enabled, then the netns was created by the OCI runtime // on controller.Start(). The OCI runtime needs to create the netns diff --git a/internal/cri/store/sandbox/sandbox.go b/internal/cri/store/sandbox/sandbox.go index ee8877f44..38458ad56 100644 --- a/internal/cri/store/sandbox/sandbox.go +++ b/internal/cri/store/sandbox/sandbox.go @@ -47,6 +47,18 @@ type Sandbox struct { *store.StopCh // Stats contains (mutable) stats for the (pause) sandbox container Stats *stats.ContainerStats + // Endpoint is the sandbox endpoint, for task or streaming api connection + Endpoint Endpoint +} + +type Endpoint struct { + Address string + Protocol string + Version uint32 +} + +func (e *Endpoint) IsValid() bool { + return e.Protocol != "" && e.Address != "" } // NewSandbox creates an internally used sandbox type. This functions reminds diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 40c77cfc0..8407468bd 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -188,10 +188,13 @@ func (c *controllerLocal) Start(ctx context.Context, sandboxID string) (sandbox. c.cleanupShim(ctx, sandboxID, svc) return sandbox.ControllerInstance{}, fmt.Errorf("failed to start sandbox %s: %w", sandboxID, errdefs.FromGRPC(err)) } - + address, protocol, version := shim.Endpoint() return sandbox.ControllerInstance{ SandboxID: sandboxID, Pid: resp.GetPid(), + Address: address, + Protocol: protocol, + Version: uint32(version), CreatedAt: resp.GetCreatedAt().AsTime(), }, nil } @@ -302,6 +305,12 @@ func (c *controllerLocal) Status(ctx context.Context, sandboxID string, verbose return sandbox.ControllerStatus{}, fmt.Errorf("failed to query sandbox %s status: %w", sandboxID, err) } + shim, err := c.shims.Get(ctx, sandboxID) + if err != nil { + return sandbox.ControllerStatus{}, fmt.Errorf("unable to find sandbox %q", sandboxID) + } + address, protocol, version := shim.Endpoint() + return sandbox.ControllerStatus{ SandboxID: resp.GetSandboxID(), Pid: resp.GetPid(), @@ -310,6 +319,9 @@ func (c *controllerLocal) Status(ctx context.Context, sandboxID string, verbose CreatedAt: resp.GetCreatedAt().AsTime(), ExitedAt: resp.GetExitedAt().AsTime(), Extra: resp.GetExtra(), + Address: address, + Protocol: protocol, + Version: uint32(version), }, nil } From 72fe47b2a226f0448c91eff3a7270b6d8117b553 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Wed, 27 Mar 2024 10:11:14 +0800 Subject: [PATCH 3/5] add task api endpoint in oci proto Signed-off-by: Abel Feng --- core/runtime/v2/runc/options/next.pb.txt | 21 +++++++++++++ core/runtime/v2/runc/options/oci.pb.go | 39 ++++++++++++++++++++++-- core/runtime/v2/runc/options/oci.proto | 6 ++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/core/runtime/v2/runc/options/next.pb.txt b/core/runtime/v2/runc/options/next.pb.txt index 537c28c97..d9d1f8f7b 100755 --- a/core/runtime/v2/runc/options/next.pb.txt +++ b/core/runtime/v2/runc/options/next.pb.txt @@ -73,6 +73,27 @@ file { type: TYPE_STRING json_name: "criuWorkPath" } + field { + name: "task_api_address" + number: 12 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "taskApiAddress" + } + field { + name: "task_api_protocol" + number: 13 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "taskApiProtocol" + } + field { + name: "task_api_version" + number: 14 + label: LABEL_OPTIONAL + type: TYPE_UINT32 + json_name: "taskApiVersion" + } reserved_range { start: 8 end: 9 diff --git a/core/runtime/v2/runc/options/oci.pb.go b/core/runtime/v2/runc/options/oci.pb.go index d68515d2b..b27007d93 100644 --- a/core/runtime/v2/runc/options/oci.pb.go +++ b/core/runtime/v2/runc/options/oci.pb.go @@ -45,6 +45,12 @@ type Options struct { CriuImagePath string `protobuf:"bytes,10,opt,name=criu_image_path,json=criuImagePath,proto3" json:"criu_image_path,omitempty"` // criu work path CriuWorkPath string `protobuf:"bytes,11,opt,name=criu_work_path,json=criuWorkPath,proto3" json:"criu_work_path,omitempty"` + // task api address, can be a unix domain socket, or vsock address. + TaskApiAddress string `protobuf:"bytes,12,opt,name=task_api_address,json=taskApiAddress,proto3" json:"task_api_address,omitempty"` + // protocol for task api connection, currently ttrpc and grpc supported + TaskApiProtocol string `protobuf:"bytes,13,opt,name=task_api_protocol,json=taskApiProtocol,proto3" json:"task_api_protocol,omitempty"` + // task api version, currently supported value is 2 and 3. + TaskApiVersion uint32 `protobuf:"varint,14,opt,name=task_api_version,json=taskApiVersion,proto3" json:"task_api_version,omitempty"` } func (x *Options) Reset() { @@ -149,6 +155,27 @@ func (x *Options) GetCriuWorkPath() string { return "" } +func (x *Options) GetTaskApiAddress() string { + if x != nil { + return x.TaskApiAddress + } + return "" +} + +func (x *Options) GetTaskApiProtocol() string { + if x != nil { + return x.TaskApiProtocol + } + return "" +} + +func (x *Options) GetTaskApiVersion() uint32 { + if x != nil { + return x.TaskApiVersion + } + return 0 +} + type CheckpointOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -325,7 +352,7 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x6f, 0x63, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xd2, 0x02, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xd2, 0x03, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x6f, 0x5f, 0x70, 0x69, 0x76, 0x6f, 0x74, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x6e, 0x6f, 0x50, 0x69, 0x76, 0x6f, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x24, 0x0a, @@ -346,7 +373,15 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x72, 0x69, 0x75, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x72, 0x69, 0x75, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x63, 0x72, 0x69, 0x75, 0x57, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x4a, 0x04, 0x08, 0x08, + 0x63, 0x72, 0x69, 0x75, 0x57, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x28, 0x0a, 0x10, + 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, + 0x70, 0x69, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x61, + 0x73, 0x6b, 0x41, 0x70, 0x69, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0xbb, 0x02, 0x0a, 0x11, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x78, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, diff --git a/core/runtime/v2/runc/options/oci.proto b/core/runtime/v2/runc/options/oci.proto index 637cd31d6..c9b5ecca8 100644 --- a/core/runtime/v2/runc/options/oci.proto +++ b/core/runtime/v2/runc/options/oci.proto @@ -29,6 +29,12 @@ message Options { string criu_image_path = 10; // criu work path string criu_work_path = 11; + // task api address, can be a unix domain socket, or vsock address. + string task_api_address = 12; + // protocol for task api connection, currently ttrpc and grpc supported + string task_api_protocol = 13; + // task api version, currently supported value is 2 and 3. + uint32 task_api_version = 14; } message CheckpointOptions { From c3b306240ead2e4d1c5bb7adf5b32aa69f5432f7 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Wed, 27 Mar 2024 11:21:12 +0800 Subject: [PATCH 4/5] add task api endpoint in task create options Signed-off-by: Abel Feng --- client/task_opts.go | 18 ++++++++++++ core/runtime/runtime.go | 6 ++++ core/runtime/v2/shim_manager.go | 36 +++++++++++++++++------ internal/cri/server/container_start.go | 2 +- plugins/services/tasks/local.go | 40 ++++++++++++++++---------- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/client/task_opts.go b/client/task_opts.go index ea6658192..82c78bc11 100644 --- a/client/task_opts.go +++ b/client/task_opts.go @@ -50,6 +50,24 @@ func WithRuntimePath(absRuntimePath string) NewTaskOpts { } } +// WithTaskAPIEndpoint allow task service to manage a task through a given endpoint, +// usually it is served inside a sandbox, and we can get it from sandbox status. +func WithTaskAPIEndpoint(address, protocol string, version uint32) NewTaskOpts { + return func(ctx context.Context, client *Client, info *TaskInfo) error { + if info.Options == nil { + info.Options = &options.Options{} + } + opts, ok := info.Options.(*options.Options) + if !ok { + return errors.New("invalid runtime v2 options format") + } + opts.TaskApiAddress = address + opts.TaskApiProtocol = protocol + opts.TaskApiVersion = version + return nil + } +} + // WithTaskCheckpoint allows a task to be created with live runtime and memory data from a // previous checkpoint. Additional software such as CRIU may be required to // restore a task from a checkpoint diff --git a/core/runtime/runtime.go b/core/runtime/runtime.go index 8c79c5c57..fdb683456 100644 --- a/core/runtime/runtime.go +++ b/core/runtime/runtime.go @@ -51,6 +51,12 @@ type CreateOpts struct { Runtime string // SandboxID is an optional ID of sandbox this container belongs to SandboxID string + // Address is an optional Address for Task API server + Address string + // Protocol is an optional Protocol for Task API connection + Protocol string + // Version is an optional Version of the Task API + Version uint32 } // Exit information for a process diff --git a/core/runtime/v2/shim_manager.go b/core/runtime/v2/shim_manager.go index 8e6b5b91a..d563d898b 100644 --- a/core/runtime/v2/shim_manager.go +++ b/core/runtime/v2/shim_manager.go @@ -163,9 +163,22 @@ func (m *ShimManager) ID() string { func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts runtime.CreateOpts) (_ ShimInstance, retErr error) { // This container belongs to sandbox which supposed to be already started via sandbox API. if opts.SandboxID != "" { - process, err := m.Get(ctx, opts.SandboxID) - if err != nil { - return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID) + var params shimbinary.BootstrapParams + if opts.Address != "" && opts.Protocol != "" { + params = shimbinary.BootstrapParams{ + Version: int(opts.Version), + Address: opts.Address, + Protocol: opts.Protocol, + } + } else { + // For those sandbox we can not get endpoint, + // fallback to legacy implementation + p, restoreErr := m.restoreBootstrapParams(ctx, opts.SandboxID) + if restoreErr != nil { + return nil, fmt.Errorf("failed to get bootstrap "+ + "params of sandbox %s, %v, legacy restore error %v", opts.SandboxID, err, restoreErr) + } + params = p } // Write sandbox ID this task belongs to. @@ -173,11 +186,6 @@ func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts return nil, err } - params, err := restoreBootstrapParams(process.Bundle()) - if err != nil { - return nil, err - } - if err := writeBootstrapParams(filepath.Join(bundle.Path, "bootstrap.json"), params); err != nil { return nil, fmt.Errorf("failed to write bootstrap.json for bundle %s: %w", bundle.Path, err) } @@ -251,6 +259,18 @@ func (m *ShimManager) startShim(ctx context.Context, bundle *Bundle, id string, return shim, nil } +func (m *ShimManager) restoreBootstrapParams(ctx context.Context, sandboxID string) (shimbinary.BootstrapParams, error) { + process, err := m.Get(ctx, sandboxID) + if err != nil { + return shimbinary.BootstrapParams{}, fmt.Errorf("can't find sandbox %s", sandboxID) + } + params, err := restoreBootstrapParams(filepath.Join(m.state, process.Namespace(), sandboxID)) + if err != nil { + return shimbinary.BootstrapParams{}, err + } + return params, nil +} + // restoreBootstrapParams reads bootstrap.json to restore shim configuration. // If its an old shim, this will perform migration - read address file and write default bootstrap // configuration (version = 2, protocol = ttrpc, and address). diff --git a/internal/cri/server/container_start.go b/internal/cri/server/container_start.go index 701339553..80c1154c9 100644 --- a/internal/cri/server/container_start.go +++ b/internal/cri/server/container_start.go @@ -123,7 +123,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain endpoint := sandbox.Endpoint if endpoint.IsValid() { taskOpts = append(taskOpts, - containerd.WithTaskApiEndpoint(endpoint.Address, endpoint.Protocol, endpoint.Version)) + containerd.WithTaskAPIEndpoint(endpoint.Address, endpoint.Protocol, endpoint.Version)) } task, err := container.NewTask(ctx, ioCreation, taskOpts...) diff --git a/plugins/services/tasks/local.go b/plugins/services/tasks/local.go index 514898c2e..fc703be3e 100644 --- a/plugins/services/tasks/local.go +++ b/plugins/services/tasks/local.go @@ -155,10 +155,25 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. if err != nil { return nil, errdefs.ToGRPC(err) } - checkpointPath, err := getRestorePath(container.Runtime.Name, r.Options) - if err != nil { - return nil, err + + var ( + checkpointPath string + taskAPIAddress string + taskAPIProtocol string + taskAPIVersion uint32 + ) + + if r.Options != nil { + taskOptions, err := formatOptions(container.Runtime.Name, r.Options) + if err != nil { + return nil, err + } + checkpointPath = taskOptions.CriuImagePath + taskAPIAddress = taskOptions.TaskApiAddress + taskAPIProtocol = taskOptions.TaskApiProtocol + taskAPIVersion = taskOptions.TaskApiVersion } + // jump get checkpointPath from checkpoint image if checkpointPath == "" && r.Checkpoint != nil { checkpointPath, err = os.MkdirTemp(os.Getenv("XDG_RUNTIME_DIR"), "ctrd-checkpoint") @@ -196,6 +211,9 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. RuntimeOptions: container.Runtime.Options, TaskOptions: r.Options, SandboxID: container.SandboxID, + Address: taskAPIAddress, + Protocol: taskAPIProtocol, + Version: taskAPIVersion, } if r.RuntimePath != "" { opts.Runtime = r.RuntimePath @@ -723,22 +741,14 @@ func getCheckpointPath(runtime string, option *ptypes.Any) (string, error) { return checkpointPath, nil } -// getRestorePath only suitable for runc runtime now -func getRestorePath(runtime string, option *ptypes.Any) (string, error) { - if option == nil { - return "", nil - } - - var restorePath string +func formatOptions(runtime string, option *ptypes.Any) (*options.Options, error) { v, err := typeurl.UnmarshalAny(option) if err != nil { - return "", err + return nil, err } opts, ok := v.(*options.Options) if !ok { - return "", fmt.Errorf("invalid task create option for %s", runtime) + return nil, fmt.Errorf("invalid task create option for %s", runtime) } - restorePath = opts.CriuImagePath - - return restorePath, nil + return opts, nil } From de38490ed6b8d9cb9cb6ea7cfcd0071e8f27d304 Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Tue, 23 Apr 2024 14:28:01 +0800 Subject: [PATCH 5/5] sandbox: merge address and protocol to one url Signed-off-by: Abel Feng --- api/next.pb.txt | 14 - api/services/sandbox/v1/sandbox.pb.go | 399 +++++++++++------------ api/services/sandbox/v1/sandbox.proto | 8 +- client/task_opts.go | 3 +- core/runtime/runtime.go | 2 - core/runtime/v2/binary.go | 4 +- core/runtime/v2/runc/options/next.pb.txt | 9 +- core/runtime/v2/runc/options/oci.pb.go | 80 ++--- core/runtime/v2/runc/options/oci.proto | 5 +- core/runtime/v2/shim.go | 23 +- core/runtime/v2/shim_manager.go | 31 +- core/sandbox/controller.go | 2 - core/sandbox/proxy/controller.go | 2 - internal/cri/server/container_start.go | 2 +- internal/cri/server/restart.go | 1 - internal/cri/server/sandbox_run.go | 7 +- internal/cri/store/sandbox/sandbox.go | 7 +- plugins/sandbox/controller.go | 6 +- plugins/services/tasks/local.go | 9 +- 19 files changed, 281 insertions(+), 333 deletions(-) diff --git a/api/next.pb.txt b/api/next.pb.txt index aa77f2f00..89c630254 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -5520,13 +5520,6 @@ file { type: TYPE_UINT32 json_name: "version" } - field { - name: "protocol" - number: 7 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "protocol" - } nested_type { name: "LabelsEntry" field { @@ -5731,13 +5724,6 @@ file { type: TYPE_UINT32 json_name: "version" } - field { - name: "protocol" - number: 10 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "protocol" - } nested_type { name: "InfoEntry" field { diff --git a/api/services/sandbox/v1/sandbox.pb.go b/api/services/sandbox/v1/sandbox.pb.go index 52e20826a..935c4e595 100644 --- a/api/services/sandbox/v1/sandbox.pb.go +++ b/api/services/sandbox/v1/sandbox.pb.go @@ -713,9 +713,11 @@ type ControllerStartResponse struct { Pid uint32 `protobuf:"varint,2,opt,name=pid,proto3" json:"pid,omitempty"` CreatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` Labels map[string]string `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Address string `protobuf:"bytes,5,opt,name=address,proto3" json:"address,omitempty"` - Version uint32 `protobuf:"varint,6,opt,name=version,proto3" json:"version,omitempty"` - Protocol string `protobuf:"bytes,7,opt,name=protocol,proto3" json:"protocol,omitempty"` + // Address of the sandbox for containerd to connect, + // for calling Task or other APIs serving in the sandbox. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. + Address string `protobuf:"bytes,5,opt,name=address,proto3" json:"address,omitempty"` + Version uint32 `protobuf:"varint,6,opt,name=version,proto3" json:"version,omitempty"` } func (x *ControllerStartResponse) Reset() { @@ -792,13 +794,6 @@ func (x *ControllerStartResponse) GetVersion() uint32 { return 0 } -func (x *ControllerStartResponse) GetProtocol() string { - if x != nil { - return x.Protocol - } - return "" -} - type ControllerPlatformRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1187,9 +1182,11 @@ type ControllerStatusResponse struct { CreatedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` ExitedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=exited_at,json=exitedAt,proto3" json:"exited_at,omitempty"` Extra *anypb.Any `protobuf:"bytes,7,opt,name=extra,proto3" json:"extra,omitempty"` - Address string `protobuf:"bytes,8,opt,name=address,proto3" json:"address,omitempty"` - Version uint32 `protobuf:"varint,9,opt,name=version,proto3" json:"version,omitempty"` - Protocol string `protobuf:"bytes,10,opt,name=protocol,proto3" json:"protocol,omitempty"` + // Address of the sandbox for containerd to connect, + // for calling Task or other APIs serving in the sandbox. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. + Address string `protobuf:"bytes,8,opt,name=address,proto3" json:"address,omitempty"` + Version uint32 `protobuf:"varint,9,opt,name=version,proto3" json:"version,omitempty"` } func (x *ControllerStatusResponse) Reset() { @@ -1287,13 +1284,6 @@ func (x *ControllerStatusResponse) GetVersion() uint32 { return 0 } -func (x *ControllerStatusResponse) GetProtocol() string { - if x != nil { - return x.Protocol - } - return "" -} - type ControllerShutdownRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1590,7 +1580,7 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, - 0x72, 0x22, 0xed, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x72, 0x22, 0xd1, 0x02, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, @@ -1607,205 +1597,202 @@ var file_github_com_containerd_containerd_api_services_sandbox_v1_sandbox_proto_ 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x58, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, - 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x54, 0x0a, 0x1a, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, - 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x08, 0x70, 0x6c, 0x61, - 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, - 0x6d, 0x22, 0x77, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, - 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x73, 0x12, 0x1c, 0x0a, 0x09, - 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x18, 0x0a, 0x16, 0x43, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x54, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, - 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, - 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x72, 0x0a, 0x16, 0x43, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x78, 0x69, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, - 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x70, - 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, - 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x62, - 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, 0x65, 0x72, 0x62, 0x6f, - 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, - 0x22, 0xe2, 0x03, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, - 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, - 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x14, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x56, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, - 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x39, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, 0x41, 0x74, - 0x12, 0x2a, 0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x12, 0x18, 0x0a, 0x07, - 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x1a, 0x37, 0x0a, 0x09, - 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x58, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, - 0x1c, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, - 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x57, 0x0a, - 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, - 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, - 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, - 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x4f, 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, - 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x32, 0xb7, 0x04, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, - 0x65, 0x12, 0x71, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x32, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, - 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6b, 0x0a, 0x04, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, - 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x2f, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x32, 0xf6, 0x07, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x12, 0x7b, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x78, 0x0a, - 0x05, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x54, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, + 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0x77, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x18, + 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x54, 0x0a, 0x15, 0x43, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x72, + 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x69, 0x74, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, + 0x78, 0x69, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x78, 0x69, + 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x65, 0x64, + 0x41, 0x74, 0x22, 0x70, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, + 0x76, 0x65, 0x72, 0x62, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, + 0x65, 0x72, 0x62, 0x6f, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, + 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, + 0x6f, 0x78, 0x65, 0x72, 0x22, 0xc6, 0x03, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, + 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x56, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, + 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x65, + 0x78, 0x69, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, + 0x65, 0x64, 0x41, 0x74, 0x12, 0x2a, 0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, + 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x37, 0x0a, 0x09, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x58, 0x0a, + 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, + 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x1c, 0x0a, 0x1a, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x57, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x49, 0x64, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x65, 0x72, 0x22, 0x4f, + 0x0a, 0x19, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x32, + 0xb7, 0x04, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x71, 0x0a, 0x06, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, + 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x71, 0x0a, 0x06, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, + 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x71, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x6b, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x30, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, + 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x68, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x47, 0x65, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xf6, 0x07, 0x0a, 0x0a, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x7b, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x78, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x81, 0x01, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, - 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x39, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, - 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, - 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x3a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, - 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x04, 0x53, - 0x74, 0x6f, 0x70, 0x12, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x75, 0x0a, 0x04, 0x57, 0x61, 0x69, 0x74, 0x12, 0x35, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x63, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x81, 0x01, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x39, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x81, 0x01, 0x0a, 0x08, 0x53, 0x68, 0x75, 0x74, 0x64, - 0x6f, 0x77, 0x6e, 0x12, 0x39, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, - 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, - 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7e, 0x0a, 0x07, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x38, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, - 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x39, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x45, 0x5a, 0x43, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, - 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, - 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, - 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x04, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x35, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, + 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x04, 0x57, 0x61, + 0x69, 0x74, 0x12, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, + 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x57, 0x61, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x37, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, + 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x81, + 0x01, 0x0a, 0x08, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x12, 0x39, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, + 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, + 0x65, 0x72, 0x53, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x7e, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x38, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x61, + 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6c, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x45, 0x5a, 0x43, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x2f, 0x76, + 0x31, 0x3b, 0x73, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/api/services/sandbox/v1/sandbox.proto b/api/services/sandbox/v1/sandbox.proto index 28de269d4..30bec54ea 100755 --- a/api/services/sandbox/v1/sandbox.proto +++ b/api/services/sandbox/v1/sandbox.proto @@ -120,9 +120,11 @@ message ControllerStartResponse { uint32 pid = 2; google.protobuf.Timestamp created_at = 3; map labels = 4; + // Address of the sandbox for containerd to connect, + // for calling Task or other APIs serving in the sandbox. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. string address = 5; uint32 version = 6; - string protocol = 7; } message ControllerPlatformRequest { @@ -166,9 +168,11 @@ message ControllerStatusResponse { google.protobuf.Timestamp created_at = 5; google.protobuf.Timestamp exited_at = 6; google.protobuf.Any extra = 7; + // Address of the sandbox for containerd to connect, + // for calling Task or other APIs serving in the sandbox. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. string address = 8; uint32 version = 9; - string protocol = 10; } message ControllerShutdownRequest { diff --git a/client/task_opts.go b/client/task_opts.go index 82c78bc11..73aac3907 100644 --- a/client/task_opts.go +++ b/client/task_opts.go @@ -52,7 +52,7 @@ func WithRuntimePath(absRuntimePath string) NewTaskOpts { // WithTaskAPIEndpoint allow task service to manage a task through a given endpoint, // usually it is served inside a sandbox, and we can get it from sandbox status. -func WithTaskAPIEndpoint(address, protocol string, version uint32) NewTaskOpts { +func WithTaskAPIEndpoint(address string, version uint32) NewTaskOpts { return func(ctx context.Context, client *Client, info *TaskInfo) error { if info.Options == nil { info.Options = &options.Options{} @@ -62,7 +62,6 @@ func WithTaskAPIEndpoint(address, protocol string, version uint32) NewTaskOpts { return errors.New("invalid runtime v2 options format") } opts.TaskApiAddress = address - opts.TaskApiProtocol = protocol opts.TaskApiVersion = version return nil } diff --git a/core/runtime/runtime.go b/core/runtime/runtime.go index fdb683456..438752f76 100644 --- a/core/runtime/runtime.go +++ b/core/runtime/runtime.go @@ -53,8 +53,6 @@ type CreateOpts struct { SandboxID string // Address is an optional Address for Task API server Address string - // Protocol is an optional Protocol for Task API connection - Protocol string // Version is an optional Version of the Task API Version uint32 } diff --git a/core/runtime/v2/binary.go b/core/runtime/v2/binary.go index e6e8eca49..9ccc9d29d 100644 --- a/core/runtime/v2/binary.go +++ b/core/runtime/v2/binary.go @@ -143,10 +143,12 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ if err := writeBootstrapParams(filepath.Join(b.bundle.Path, "bootstrap.json"), params); err != nil { return nil, fmt.Errorf("failed to write bootstrap.json: %w", err) } - + // The address is in the form like ttrpc+unix:// or grpc+vsock://: + address := fmt.Sprintf("%s+%s", params.Protocol, params.Address) return &shim{ bundle: b.bundle, client: conn, + address: address, version: params.Version, }, nil } diff --git a/core/runtime/v2/runc/options/next.pb.txt b/core/runtime/v2/runc/options/next.pb.txt index d9d1f8f7b..6b5ce784b 100755 --- a/core/runtime/v2/runc/options/next.pb.txt +++ b/core/runtime/v2/runc/options/next.pb.txt @@ -80,16 +80,9 @@ file { type: TYPE_STRING json_name: "taskApiAddress" } - field { - name: "task_api_protocol" - number: 13 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "taskApiProtocol" - } field { name: "task_api_version" - number: 14 + number: 13 label: LABEL_OPTIONAL type: TYPE_UINT32 json_name: "taskApiVersion" diff --git a/core/runtime/v2/runc/options/oci.pb.go b/core/runtime/v2/runc/options/oci.pb.go index b27007d93..89681c121 100644 --- a/core/runtime/v2/runc/options/oci.pb.go +++ b/core/runtime/v2/runc/options/oci.pb.go @@ -46,11 +46,10 @@ type Options struct { // criu work path CriuWorkPath string `protobuf:"bytes,11,opt,name=criu_work_path,json=criuWorkPath,proto3" json:"criu_work_path,omitempty"` // task api address, can be a unix domain socket, or vsock address. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. TaskApiAddress string `protobuf:"bytes,12,opt,name=task_api_address,json=taskApiAddress,proto3" json:"task_api_address,omitempty"` - // protocol for task api connection, currently ttrpc and grpc supported - TaskApiProtocol string `protobuf:"bytes,13,opt,name=task_api_protocol,json=taskApiProtocol,proto3" json:"task_api_protocol,omitempty"` // task api version, currently supported value is 2 and 3. - TaskApiVersion uint32 `protobuf:"varint,14,opt,name=task_api_version,json=taskApiVersion,proto3" json:"task_api_version,omitempty"` + TaskApiVersion uint32 `protobuf:"varint,13,opt,name=task_api_version,json=taskApiVersion,proto3" json:"task_api_version,omitempty"` } func (x *Options) Reset() { @@ -162,13 +161,6 @@ func (x *Options) GetTaskApiAddress() string { return "" } -func (x *Options) GetTaskApiProtocol() string { - if x != nil { - return x.TaskApiProtocol - } - return "" -} - func (x *Options) GetTaskApiVersion() uint32 { if x != nil { return x.TaskApiVersion @@ -352,7 +344,7 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x6f, 0x63, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xd2, 0x03, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xa6, 0x03, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x6f, 0x5f, 0x70, 0x69, 0x76, 0x6f, 0x74, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x6e, 0x6f, 0x50, 0x69, 0x76, 0x6f, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x24, 0x0a, @@ -376,40 +368,38 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto 0x63, 0x72, 0x69, 0x75, 0x57, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x41, - 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, - 0x70, 0x69, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x61, - 0x73, 0x6b, 0x41, 0x70, 0x69, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x08, - 0x10, 0x09, 0x22, 0xbb, 0x02, 0x0a, 0x11, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x78, 0x69, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x6f, 0x70, 0x65, 0x6e, 0x5f, 0x74, 0x63, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, - 0x6f, 0x70, 0x65, 0x6e, 0x54, 0x63, 0x70, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x55, 0x6e, 0x69, 0x78, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x74, - 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x74, - 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x66, 0x69, 0x6c, - 0x65, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x5f, 0x6d, 0x6f, 0x64, - 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, - 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x50, - 0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, - 0x22, 0x29, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x78, 0x65, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x78, 0x65, 0x63, 0x49, 0x64, 0x42, 0x4a, 0x5a, 0x48, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, - 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, - 0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3b, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, + 0x70, 0x69, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0xbb, 0x02, 0x0a, 0x11, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x65, 0x78, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x70, 0x65, 0x6e, 0x5f, 0x74, 0x63, 0x70, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x07, 0x6f, 0x70, 0x65, 0x6e, 0x54, 0x63, 0x70, 0x12, 0x32, 0x0a, 0x15, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x73, 0x6f, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x55, 0x6e, 0x69, 0x78, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, + 0x1a, 0x0a, 0x08, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x08, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x66, 0x69, 0x6c, 0x65, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6d, + 0x70, 0x74, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, + 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x73, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67, + 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, + 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x5f, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, + 0x50, 0x61, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x78, 0x65, 0x63, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x78, 0x65, 0x63, 0x49, 0x64, 0x42, + 0x4a, 0x5a, 0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x3b, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/core/runtime/v2/runc/options/oci.proto b/core/runtime/v2/runc/options/oci.proto index c9b5ecca8..3273f012d 100644 --- a/core/runtime/v2/runc/options/oci.proto +++ b/core/runtime/v2/runc/options/oci.proto @@ -30,11 +30,10 @@ message Options { // criu work path string criu_work_path = 11; // task api address, can be a unix domain socket, or vsock address. + // it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://:. string task_api_address = 12; - // protocol for task api connection, currently ttrpc and grpc supported - string task_api_protocol = 13; // task api version, currently supported value is 2 and 3. - uint32 task_api_version = 14; + uint32 task_api_version = 13; } message CheckpointOptions { diff --git a/core/runtime/v2/shim.go b/core/runtime/v2/shim.go index 89414bc43..27676ed0d 100644 --- a/core/runtime/v2/shim.go +++ b/core/runtime/v2/shim.go @@ -113,9 +113,13 @@ func loadShim(ctx context.Context, bundle *Bundle, onClose func()) (_ ShimInstan } }() + // The address is in the form like ttrpc+unix:// or grpc+vsock://: + address := fmt.Sprintf("%s+%s", params.Protocol, params.Address) + shim := &shim{ bundle: bundle, client: conn, + address: address, version: params.Version, } @@ -186,8 +190,8 @@ type ShimInstance interface { // Delete will close the client and remove bundle from disk. Delete(ctx context.Context) error // Endpoint returns shim's endpoint information, - // including address, protocol and version. - Endpoint() (string, string, int) + // including address and version. + Endpoint() (string, int) } func parseStartResponse(response []byte) (client.BootstrapParams, error) { @@ -360,11 +364,10 @@ func (gc *grpcConn) UserOnCloseWait(ctx context.Context) error { } type shim struct { - bundle *Bundle - client any - address string - protocol string - version int + bundle *Bundle + client any + address string + version int } var _ ShimInstance = (*shim)(nil) @@ -374,8 +377,8 @@ func (s *shim) ID() string { return s.bundle.ID } -func (s *shim) Endpoint() (string, string, int) { - return s.address, s.protocol, s.version +func (s *shim) Endpoint() (string, int) { + return s.address, s.version } func (s *shim) Namespace() string { @@ -443,7 +446,7 @@ type shimTask struct { } func newShimTask(shim ShimInstance) (*shimTask, error) { - _, _, version := shim.Endpoint() + _, version := shim.Endpoint() taskClient, err := NewTaskClient(shim.Client(), version) if err != nil { return nil, err diff --git a/core/runtime/v2/shim_manager.go b/core/runtime/v2/shim_manager.go index d563d898b..e254934e3 100644 --- a/core/runtime/v2/shim_manager.go +++ b/core/runtime/v2/shim_manager.go @@ -164,16 +164,27 @@ func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts // This container belongs to sandbox which supposed to be already started via sandbox API. if opts.SandboxID != "" { var params shimbinary.BootstrapParams - if opts.Address != "" && opts.Protocol != "" { + if opts.Address != "" { + // The address returned from sandbox controller should be in the form like ttrpc+unix:// + // or grpc+vsock://:, we should get the protocol from the url first. + protocol, address, ok := strings.Cut(opts.Address, "+") + if !ok { + return nil, fmt.Errorf("the scheme of sandbox address should be in " + + " the form of +, i.e. ttrpc+unix or grpc+vsock") + } params = shimbinary.BootstrapParams{ Version: int(opts.Version), - Address: opts.Address, - Protocol: opts.Protocol, + Protocol: protocol, + Address: address, } } else { // For those sandbox we can not get endpoint, // fallback to legacy implementation - p, restoreErr := m.restoreBootstrapParams(ctx, opts.SandboxID) + process, err := m.Get(ctx, opts.SandboxID) + if err != nil { + return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID) + } + p, restoreErr := restoreBootstrapParams(process.Bundle()) if restoreErr != nil { return nil, fmt.Errorf("failed to get bootstrap "+ "params of sandbox %s, %v, legacy restore error %v", opts.SandboxID, err, restoreErr) @@ -259,18 +270,6 @@ func (m *ShimManager) startShim(ctx context.Context, bundle *Bundle, id string, return shim, nil } -func (m *ShimManager) restoreBootstrapParams(ctx context.Context, sandboxID string) (shimbinary.BootstrapParams, error) { - process, err := m.Get(ctx, sandboxID) - if err != nil { - return shimbinary.BootstrapParams{}, fmt.Errorf("can't find sandbox %s", sandboxID) - } - params, err := restoreBootstrapParams(filepath.Join(m.state, process.Namespace(), sandboxID)) - if err != nil { - return shimbinary.BootstrapParams{}, err - } - return params, nil -} - // restoreBootstrapParams reads bootstrap.json to restore shim configuration. // If its an old shim, this will perform migration - read address file and write default bootstrap // configuration (version = 2, protocol = ttrpc, and address). diff --git a/core/sandbox/controller.go b/core/sandbox/controller.go index cf9a4302c..a95abd778 100644 --- a/core/sandbox/controller.go +++ b/core/sandbox/controller.go @@ -118,7 +118,6 @@ type ControllerInstance struct { Pid uint32 CreatedAt time.Time Address string - Protocol string Version uint32 Labels map[string]string } @@ -137,6 +136,5 @@ type ControllerStatus struct { ExitedAt time.Time Extra typeurl.Any Address string - Protocol string Version uint32 } diff --git a/core/sandbox/proxy/controller.go b/core/sandbox/proxy/controller.go index 661381367..fa846af6f 100644 --- a/core/sandbox/proxy/controller.go +++ b/core/sandbox/proxy/controller.go @@ -74,7 +74,6 @@ func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) ( CreatedAt: resp.GetCreatedAt().AsTime(), Labels: resp.GetLabels(), Address: resp.GetAddress(), - Protocol: resp.GetProtocol(), Version: resp.GetVersion(), }, nil } @@ -145,7 +144,6 @@ func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string, ExitedAt: resp.GetExitedAt().AsTime(), Extra: resp.GetExtra(), Address: resp.GetAddress(), - Protocol: resp.GetProtocol(), Version: resp.GetVersion(), }, nil } diff --git a/internal/cri/server/container_start.go b/internal/cri/server/container_start.go index 80c1154c9..173f2468b 100644 --- a/internal/cri/server/container_start.go +++ b/internal/cri/server/container_start.go @@ -123,7 +123,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain endpoint := sandbox.Endpoint if endpoint.IsValid() { taskOpts = append(taskOpts, - containerd.WithTaskAPIEndpoint(endpoint.Address, endpoint.Protocol, endpoint.Version)) + containerd.WithTaskAPIEndpoint(endpoint.Address, endpoint.Version)) } task, err := container.NewTask(ctx, ioCreation, taskOpts...) diff --git a/internal/cri/server/restart.go b/internal/cri/server/restart.go index e95698885..70f433570 100644 --- a/internal/cri/server/restart.go +++ b/internal/cri/server/restart.go @@ -127,7 +127,6 @@ func (c *criService) recover(ctx context.Context) error { state = sandboxstore.StateNotReady } } else { - endpoint.Protocol = status.Protocol endpoint.Version = status.Version endpoint.Address = status.Address if code, ok := runtime.PodSandboxState_value[status.State]; ok { diff --git a/internal/cri/server/sandbox_run.go b/internal/cri/server/sandbox_run.go index 4f5fda925..0157d4b7d 100644 --- a/internal/cri/server/sandbox_run.go +++ b/internal/cri/server/sandbox_run.go @@ -265,11 +265,10 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("failed to start sandbox %q: %w", id, err) } - if ctrl.Protocol != "" && ctrl.Address != "" { + if ctrl.Address != "" { sandbox.Endpoint = sandboxstore.Endpoint{ - Protocol: ctrl.Protocol, - Version: ctrl.Version, - Address: ctrl.Address, + Version: ctrl.Version, + Address: ctrl.Address, } } diff --git a/internal/cri/store/sandbox/sandbox.go b/internal/cri/store/sandbox/sandbox.go index 38458ad56..a26e94f02 100644 --- a/internal/cri/store/sandbox/sandbox.go +++ b/internal/cri/store/sandbox/sandbox.go @@ -52,13 +52,12 @@ type Sandbox struct { } type Endpoint struct { - Address string - Protocol string - Version uint32 + Address string + Version uint32 } func (e *Endpoint) IsValid() bool { - return e.Protocol != "" && e.Address != "" + return e.Address != "" } // NewSandbox creates an internally used sandbox type. This functions reminds diff --git a/plugins/sandbox/controller.go b/plugins/sandbox/controller.go index 8407468bd..01f1fb8c7 100644 --- a/plugins/sandbox/controller.go +++ b/plugins/sandbox/controller.go @@ -188,12 +188,11 @@ func (c *controllerLocal) Start(ctx context.Context, sandboxID string) (sandbox. c.cleanupShim(ctx, sandboxID, svc) return sandbox.ControllerInstance{}, fmt.Errorf("failed to start sandbox %s: %w", sandboxID, errdefs.FromGRPC(err)) } - address, protocol, version := shim.Endpoint() + address, version := shim.Endpoint() return sandbox.ControllerInstance{ SandboxID: sandboxID, Pid: resp.GetPid(), Address: address, - Protocol: protocol, Version: uint32(version), CreatedAt: resp.GetCreatedAt().AsTime(), }, nil @@ -309,7 +308,7 @@ func (c *controllerLocal) Status(ctx context.Context, sandboxID string, verbose if err != nil { return sandbox.ControllerStatus{}, fmt.Errorf("unable to find sandbox %q", sandboxID) } - address, protocol, version := shim.Endpoint() + address, version := shim.Endpoint() return sandbox.ControllerStatus{ SandboxID: resp.GetSandboxID(), @@ -320,7 +319,6 @@ func (c *controllerLocal) Status(ctx context.Context, sandboxID string, verbose ExitedAt: resp.GetExitedAt().AsTime(), Extra: resp.GetExtra(), Address: address, - Protocol: protocol, Version: uint32(version), }, nil } diff --git a/plugins/services/tasks/local.go b/plugins/services/tasks/local.go index fc703be3e..b61fbe691 100644 --- a/plugins/services/tasks/local.go +++ b/plugins/services/tasks/local.go @@ -157,10 +157,9 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. } var ( - checkpointPath string - taskAPIAddress string - taskAPIProtocol string - taskAPIVersion uint32 + checkpointPath string + taskAPIAddress string + taskAPIVersion uint32 ) if r.Options != nil { @@ -170,7 +169,6 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. } checkpointPath = taskOptions.CriuImagePath taskAPIAddress = taskOptions.TaskApiAddress - taskAPIProtocol = taskOptions.TaskApiProtocol taskAPIVersion = taskOptions.TaskApiVersion } @@ -212,7 +210,6 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. TaskOptions: r.Options, SandboxID: container.SandboxID, Address: taskAPIAddress, - Protocol: taskAPIProtocol, Version: taskAPIVersion, } if r.RuntimePath != "" {