github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #9736 from abel-von/sandbox-task-0201

Browse Source 
sandbox: Store bootstrap parameters in sandbox metadata and shim get them from sandbox metadata rather than other shim's bootstrap.json file.
This commit is contained in:
Fu Wei
2024-05-02 15:35:29 +00:00
committed by GitHub
parent 182a5fc653 de38490ed6
commit a91b05d99c
19 changed files with 474 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/runtime/runtime.go
View File

@@ -51,6 +51,10 @@ type CreateOpts struct {
Runtime string
// SandboxID is an optional ID of sandbox this container belongs to
SandboxID string
// Address is an optional Address for Task API server
Address string
// Version is an optional Version of the Task API
Version uint32
}
// Exit information for a process

4
core/runtime/v2/binary.go
View File

@@ -143,10 +143,12 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_
if err := writeBootstrapParams(filepath.Join(b.bundle.Path, "bootstrap.json"), params); err != nil {
return nil, fmt.Errorf("failed to write bootstrap.json: %w", err)
}
// The address is in the form like ttrpc+unix://<uds-path> or grpc+vsock://<cid>:<port>
address := fmt.Sprintf("%s+%s", params.Protocol, params.Address)
return &shim{
bundle: b.bundle,
client: conn,
address: address,
version: params.Version,
}, nil
}

14
core/runtime/v2/runc/options/next.pb.txt
View File

@@ -73,6 +73,20 @@ file {
type: TYPE_STRING
json_name: "criuWorkPath"
}
field {
name: "task_api_address"
number: 12
label: LABEL_OPTIONAL
type: TYPE_STRING
json_name: "taskApiAddress"
}
field {
name: "task_api_version"
number: 13
label: LABEL_OPTIONAL
type: TYPE_UINT32
json_name: "taskApiVersion"
}
reserved_range {
start: 8
end: 9

85
core/runtime/v2/runc/options/oci.pb.go
View File

@@ -45,6 +45,11 @@ type Options struct {
CriuImagePath string `protobuf:"bytes,10,opt,name=criu_image_path,json=criuImagePath,proto3" json:"criu_image_path,omitempty"`
// criu work path
CriuWorkPath string `protobuf:"bytes,11,opt,name=criu_work_path,json=criuWorkPath,proto3" json:"criu_work_path,omitempty"`
// task api address, can be a unix domain socket, or vsock address.
// it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>.
TaskApiAddress string `protobuf:"bytes,12,opt,name=task_api_address,json=taskApiAddress,proto3" json:"task_api_address,omitempty"`
// task api version, currently supported value is 2 and 3.
TaskApiVersion uint32 `protobuf:"varint,13,opt,name=task_api_version,json=taskApiVersion,proto3" json:"task_api_version,omitempty"`
}
func (x *Options) Reset() {
@@ -149,6 +154,20 @@ func (x *Options) GetCriuWorkPath() string {
return ""
}
func (x *Options) GetTaskApiAddress() string {
if x != nil {
return x.TaskApiAddress
}
return ""
}
func (x *Options) GetTaskApiVersion() uint32 {
if x != nil {
return x.TaskApiVersion
}
return 0
}
type CheckpointOptions struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -325,7 +344,7 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto
0x72, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f,
0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f,
0x6f, 0x63, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xd2, 0x02,
0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x72, 0x75, 0x6e, 0x63, 0x2e, 0x76, 0x31, 0x22, 0xa6, 0x03,
0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x6f, 0x5f,
0x70, 0x69, 0x76, 0x6f, 0x74, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08,
0x52, 0x0b, 0x6e, 0x6f, 0x50, 0x69, 0x76, 0x6f, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x24, 0x0a,
@@ -346,35 +365,41 @@ var file_github_com_containerd_containerd_core_runtime_v2_runc_options_oci_proto
0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x72, 0x69, 0x75, 0x49, 0x6d, 0x61,
0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x72, 0x69, 0x75, 0x5f, 0x77,
0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c,
0x63, 0x72, 0x69, 0x75, 0x57, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x4a, 0x04, 0x08, 0x08,
0x10, 0x09, 0x22, 0xbb, 0x02, 0x0a, 0x11, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e,
0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x78, 0x69, 0x74,
0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08,
0x6f, 0x70, 0x65, 0x6e, 0x5f, 0x74, 0x63, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07,
0x6f, 0x70, 0x65, 0x6e, 0x54, 0x63, 0x70, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72,
0x6e, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73,
0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
0x55, 0x6e, 0x69, 0x78, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x74,
0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x74,
0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f,
0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x66, 0x69, 0x6c,
0x65, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x5f,
0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09,
0x52, 0x0f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x5f, 0x6d, 0x6f, 0x64,
0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73,
0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x61,
0x74, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x50,
0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68,
0x22, 0x29, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69,
0x6c, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x78, 0x65, 0x63, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x78, 0x65, 0x63, 0x49, 0x64, 0x42, 0x4a, 0x5a, 0x48, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f,
0x76, 0x32, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f,
0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3b,
0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x63, 0x72, 0x69, 0x75, 0x57, 0x6f, 0x72, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x28, 0x0a, 0x10,
0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73,
0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x41,
0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x61,
0x70, 0x69, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d,
0x52, 0x0e, 0x74, 0x61, 0x73, 0x6b, 0x41, 0x70, 0x69, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0xbb, 0x02, 0x0a, 0x11, 0x43, 0x68, 0x65, 0x63, 0x6b,
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04,
0x65, 0x78, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74,
0x12, 0x19, 0x0a, 0x08, 0x6f, 0x70, 0x65, 0x6e, 0x5f, 0x74, 0x63, 0x70, 0x18, 0x02, 0x20, 0x01,
0x28, 0x08, 0x52, 0x07, 0x6f, 0x70, 0x65, 0x6e, 0x54, 0x63, 0x70, 0x12, 0x32, 0x0a, 0x15, 0x65,
0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x73, 0x6f, 0x63,
0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x78, 0x74, 0x65,
0x72, 0x6e, 0x61, 0x6c, 0x55, 0x6e, 0x69, 0x78, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12,
0x1a, 0x0a, 0x08, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28,
0x08, 0x52, 0x08, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x66,
0x69, 0x6c, 0x65, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52,
0x09, 0x66, 0x69, 0x6c, 0x65, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x6d,
0x70, 0x74, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x06,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73,
0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73,
0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x67, 0x72,
0x6f, 0x75, 0x70, 0x73, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67,
0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d,
0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x5f,
0x70, 0x61, 0x74, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b,
0x50, 0x61, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x44,
0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x78, 0x65, 0x63, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x78, 0x65, 0x63, 0x49, 0x64, 0x42,
0x4a, 0x5a, 0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f,
0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
0x65, 0x72, 0x64, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x74,
0x69, 0x6d, 0x65, 0x2f, 0x76, 0x32, 0x2f, 0x72, 0x75, 0x6e, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x3b, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (

5
core/runtime/v2/runc/options/oci.proto
View File

@@ -29,6 +29,11 @@ message Options {
string criu_image_path = 10;
// criu work path
string criu_work_path = 11;
// task api address, can be a unix domain socket, or vsock address.
// it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>.
string task_api_address = 12;
// task api version, currently supported value is 2 and 3.
uint32 task_api_version = 13;
}
message CheckpointOptions {

17
core/runtime/v2/shim.go
View File

@@ -113,9 +113,13 @@ func loadShim(ctx context.Context, bundle *Bundle, onClose func()) (_ ShimInstan
}
}()
// The address is in the form like ttrpc+unix://<uds-path> or grpc+vsock://<cid>:<port>
address := fmt.Sprintf("%s+%s", params.Protocol, params.Address)
shim := &shim{
bundle: bundle,
client: conn,
address: address,
version: params.Version,
}
@@ -185,8 +189,9 @@ type ShimInstance interface {
Client() any
// Delete will close the client and remove bundle from disk.
Delete(ctx context.Context) error
// Version returns shim's features compatibility version.
Version() int
// Endpoint returns shim's endpoint information,
// including address and version.
Endpoint() (string, int)
}
func parseStartResponse(response []byte) (client.BootstrapParams, error) {
@@ -361,6 +366,7 @@ func (gc *grpcConn) UserOnCloseWait(ctx context.Context) error {
type shim struct {
bundle *Bundle
client any
address string
version int
}
@@ -371,8 +377,8 @@ func (s *shim) ID() string {
return s.bundle.ID
}
func (s *shim) Version() int {
return s.version
func (s *shim) Endpoint() (string, int) {
return s.address, s.version
}
func (s *shim) Namespace() string {
@@ -440,7 +446,8 @@ type shimTask struct {
}
func newShimTask(shim ShimInstance) (*shimTask, error) {
taskClient, err := NewTaskClient(shim.Client(), shim.Version())
_, version := shim.Endpoint()
taskClient, err := NewTaskClient(shim.Client(), version)
if err != nil {
return nil, err
}

35
core/runtime/v2/shim_manager.go
View File

@@ -163,9 +163,33 @@ func (m *ShimManager) ID() string {
func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts runtime.CreateOpts) (_ ShimInstance, retErr error) {
// This container belongs to sandbox which supposed to be already started via sandbox API.
if opts.SandboxID != "" {
process, err := m.Get(ctx, opts.SandboxID)
if err != nil {
return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID)
var params shimbinary.BootstrapParams
if opts.Address != "" {
// The address returned from sandbox controller should be in the form like ttrpc+unix://<uds-path>
// or grpc+vsock://<cid>:<port>, we should get the protocol from the url first.
protocol, address, ok := strings.Cut(opts.Address, "+")
if !ok {
return nil, fmt.Errorf("the scheme of sandbox address should be in " +
" the form of <protocol>+<unix|vsock|tcp>, i.e. ttrpc+unix or grpc+vsock")
}
params = shimbinary.BootstrapParams{
Version: int(opts.Version),
Protocol: protocol,
Address: address,
}
} else {
// For those sandbox we can not get endpoint,
// fallback to legacy implementation
process, err := m.Get(ctx, opts.SandboxID)
if err != nil {
return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID)
}
p, restoreErr := restoreBootstrapParams(process.Bundle())
if restoreErr != nil {
return nil, fmt.Errorf("failed to get bootstrap "+
"params of sandbox %s, %v, legacy restore error %v", opts.SandboxID, err, restoreErr)
}
params = p
}
// Write sandbox ID this task belongs to.
@@ -173,11 +197,6 @@ func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts
return nil, err
}
params, err := restoreBootstrapParams(process.Bundle())
if err != nil {
return nil, err
}
if err := writeBootstrapParams(filepath.Join(bundle.Path, "bootstrap.json"), params); err != nil {
return nil, fmt.Errorf("failed to write bootstrap.json for bundle %s: %w", bundle.Path, err)
}

4
core/sandbox/controller.go
View File

@@ -117,6 +117,8 @@ type ControllerInstance struct {
SandboxID string
Pid uint32
CreatedAt time.Time
Address string
Version uint32
Labels map[string]string
}
@@ -133,4 +135,6 @@ type ControllerStatus struct {
CreatedAt time.Time
ExitedAt time.Time
Extra typeurl.Any
Address string
Version uint32
}

4
core/sandbox/proxy/controller.go
View File

@@ -73,6 +73,8 @@ func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) (
Pid: resp.GetPid(),
CreatedAt: resp.GetCreatedAt().AsTime(),
Labels: resp.GetLabels(),
Address: resp.GetAddress(),
Version: resp.GetVersion(),
}, nil
}
@@ -141,6 +143,8 @@ func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string,
CreatedAt: resp.GetCreatedAt().AsTime(),
ExitedAt: resp.GetExitedAt().AsTime(),
Extra: resp.GetExtra(),
Address: resp.GetAddress(),
Version: resp.GetVersion(),
}, nil
}