diff --git a/.gitignore b/.gitignore index 07b5e2481..4c16aaf04 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ containerd/containerd +containerd-shim/containerd-shim bin/ ctr/ctr hack/benchmark diff --git a/Makefile b/Makefile index 21619a0c4..a3adac920 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -BUILDTAGS=libcontainer +BUILDTAGS= # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach @@ -13,7 +13,7 @@ DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" export GOPATH:=$(CURDIR)/vendor:$(GOPATH) -all: client daemon +all: client daemon shim bin: mkdir -p bin/ @@ -27,6 +27,9 @@ client: bin daemon: bin cd containerd && go build -tags "$(BUILDTAGS)" -o ../bin/containerd +shim: bin + cd containerd-shim && go build -tags "$(BUILDTAGS)" -o ../bin/containerd-shim + dbuild: @docker build --rm --force-rm -t "$(DOCKER_IMAGE)" . diff --git a/api/grpc/server/server.go b/api/grpc/server/server.go index 430e00c05..e6edcb505 100644 --- a/api/grpc/server/server.go +++ b/api/grpc/server/server.go @@ -33,10 +33,6 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine e := supervisor.NewEvent(supervisor.StartContainerEventType) e.ID = c.Id e.BundlePath = c.BundlePath - e.Stdout = c.Stdout - e.Stderr = c.Stderr - e.Stdin = c.Stdin - e.Console = c.Console e.StartResponse = make(chan supervisor.StartResponse, 1) if c.Checkpoint != "" { e.Checkpoint = &runtime.Checkpoint{ @@ -49,14 +45,16 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine } sr := <-e.StartResponse return &types.CreateContainerResponse{ - Pid: uint32(sr.Pid), + Stdin: sr.Stdin, + Stdout: sr.Stdout, + Stderr: sr.Stderr, }, nil } func (s *apiServer) Signal(ctx context.Context, r *types.SignalRequest) (*types.SignalResponse, error) { e := supervisor.NewEvent(supervisor.SignalEventType) e.ID = r.Id - e.Pid = int(r.Pid) + e.Pid = r.Pid e.Signal = syscall.Signal(int(r.Signal)) s.sv.SendEvent(e) if err := <-e.Err; err != nil { @@ -79,7 +77,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest) } e := supervisor.NewEvent(supervisor.AddProcessEventType) e.ID = r.Id - e.Process = process + e.ProcessSpec = process e.Console = r.Console e.Stdin = r.Stdin e.Stdout = r.Stdout @@ -88,7 +86,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest) if err := <-e.Err; err != nil { return nil, err } - return &types.AddProcessResponse{Pid: uint32(e.Pid)}, nil + return &types.AddProcessResponse{}, nil } func (s *apiServer) CreateCheckpoint(ctx context.Context, r *types.CreateCheckpointRequest) (*types.CreateCheckpointResponse, error) { @@ -140,21 +138,23 @@ func (s *apiServer) ListCheckpoint(ctx context.Context, r *types.ListCheckpointR if container == nil { return nil, grpc.Errorf(codes.NotFound, "no such containers") } - checkpoints, err := container.Checkpoints() - if err != nil { - return nil, err - } var out []*types.Checkpoint - for _, c := range checkpoints { - out = append(out, &types.Checkpoint{ - Name: c.Name, - Tcp: c.Tcp, - Shell: c.Shell, - UnixSockets: c.UnixSockets, - // TODO: figure out timestamp - //Timestamp: c.Timestamp, - }) - } + /* + checkpoints, err := container.Checkpoints() + if err != nil { + return nil, err + } + for _, c := range checkpoints { + out = append(out, &types.Checkpoint{ + Name: c.Name, + Tcp: c.Tcp, + Shell: c.Shell, + UnixSockets: c.UnixSockets, + // TODO: figure out timestamp + //Timestamp: c.Timestamp, + }) + } + */ return &types.ListCheckpointResponse{Checkpoints: out}, nil } @@ -167,7 +167,6 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St m := s.sv.Machine() state := &types.StateResponse{ Machine: &types.Machine{ - Id: m.ID, Cpus: uint32(m.Cpus), Memory: uint64(m.Cpus), }, @@ -179,13 +178,9 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St } var procs []*types.Process for _, p := range processes { - pid, err := p.Pid() - if err != nil { - logrus.WithField("error", err).Error("get process pid") - } oldProc := p.Spec() procs = append(procs, &types.Process{ - Pid: uint32(pid), + Pid: p.ID(), Terminal: oldProc.Terminal, Args: oldProc.Args, Env: oldProc.Env, @@ -231,7 +226,7 @@ func (s *apiServer) Events(r *types.EventsRequest, stream types.API_EventsServer ev = &types.Event{ Type: "exit", Id: evt.ID, - Pid: uint32(evt.Pid), + Pid: evt.Pid, Status: uint32(evt.Status), } case supervisor.OOMEventType: diff --git a/api/grpc/types/api.pb.go b/api/grpc/types/api.pb.go index 0455a8c48..10b0e20eb 100644 --- a/api/grpc/types/api.pb.go +++ b/api/grpc/types/api.pb.go @@ -65,11 +65,7 @@ var _ = math.Inf type CreateContainerRequest struct { Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` BundlePath string `protobuf:"bytes,2,opt,name=bundlePath" json:"bundlePath,omitempty"` - Stdin string `protobuf:"bytes,3,opt,name=stdin" json:"stdin,omitempty"` - Stdout string `protobuf:"bytes,4,opt,name=stdout" json:"stdout,omitempty"` - Stderr string `protobuf:"bytes,5,opt,name=stderr" json:"stderr,omitempty"` - Console string `protobuf:"bytes,6,opt,name=console" json:"console,omitempty"` - Checkpoint string `protobuf:"bytes,7,opt,name=checkpoint" json:"checkpoint,omitempty"` + Checkpoint string `protobuf:"bytes,3,opt,name=checkpoint" json:"checkpoint,omitempty"` } func (m *CreateContainerRequest) Reset() { *m = CreateContainerRequest{} } @@ -78,7 +74,9 @@ func (*CreateContainerRequest) ProtoMessage() {} func (*CreateContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } type CreateContainerResponse struct { - Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"` + Stdin string `protobuf:"bytes,1,opt,name=stdin" json:"stdin,omitempty"` + Stdout string `protobuf:"bytes,2,opt,name=stdout" json:"stdout,omitempty"` + Stderr string `protobuf:"bytes,3,opt,name=stderr" json:"stderr,omitempty"` } func (m *CreateContainerResponse) Reset() { *m = CreateContainerResponse{} } @@ -88,7 +86,7 @@ func (*CreateContainerResponse) Descriptor() ([]byte, []int) { return fileDescri type SignalRequest struct { Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` - Pid uint32 `protobuf:"varint,2,opt,name=pid" json:"pid,omitempty"` + Pid string `protobuf:"bytes,2,opt,name=pid" json:"pid,omitempty"` Signal uint32 `protobuf:"varint,3,opt,name=signal" json:"signal,omitempty"` } @@ -142,7 +140,6 @@ func (*User) ProtoMessage() {} func (*User) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } type AddProcessResponse struct { - Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"` } func (m *AddProcessResponse) Reset() { *m = AddProcessResponse{} } @@ -249,7 +246,7 @@ func (*ContainerState) ProtoMessage() {} func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } type Process struct { - Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"` + Pid string `protobuf:"bytes,1,opt,name=pid" json:"pid,omitempty"` Terminal bool `protobuf:"varint,2,opt,name=terminal" json:"terminal,omitempty"` User *User `protobuf:"bytes,3,opt,name=user" json:"user,omitempty"` Args []string `protobuf:"bytes,4,rep,name=args" json:"args,omitempty"` @@ -291,9 +288,8 @@ func (m *Container) GetProcesses() []*Process { // Machine is information about machine on which containerd is run type Machine struct { - Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` - Cpus uint32 `protobuf:"varint,2,opt,name=cpus" json:"cpus,omitempty"` - Memory uint64 `protobuf:"varint,3,opt,name=memory" json:"memory,omitempty"` + Cpus uint32 `protobuf:"varint,1,opt,name=cpus" json:"cpus,omitempty"` + Memory uint64 `protobuf:"varint,2,opt,name=memory" json:"memory,omitempty"` } func (m *Machine) Reset() { *m = Machine{} } @@ -358,7 +354,7 @@ type Event struct { Id string `protobuf:"bytes,2,opt,name=id" json:"id,omitempty"` Status uint32 `protobuf:"varint,3,opt,name=status" json:"status,omitempty"` BundlePath string `protobuf:"bytes,4,opt,name=bundlePath" json:"bundlePath,omitempty"` - Pid uint32 `protobuf:"varint,5,opt,name=pid" json:"pid,omitempty"` + Pid string `protobuf:"bytes,5,opt,name=pid" json:"pid,omitempty"` Signal uint32 `protobuf:"varint,7,opt,name=signal" json:"signal,omitempty"` Process *Process `protobuf:"bytes,8,opt,name=process" json:"process,omitempty"` Containers []*Container `protobuf:"bytes,9,rep,name=containers" json:"containers,omitempty"` @@ -1087,96 +1083,95 @@ var _API_serviceDesc = grpc.ServiceDesc{ } var fileDescriptor0 = []byte{ - // 1454 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0x1c, 0x45, - 0x17, 0xf6, 0xcc, 0xf4, 0x6c, 0x67, 0x16, 0xdb, 0xed, 0xd8, 0x19, 0xcf, 0xff, 0x87, 0x24, 0x4d, - 0x08, 0x11, 0x8a, 0xac, 0xe0, 0xb0, 0x84, 0x70, 0x01, 0xc1, 0x89, 0x12, 0x50, 0x02, 0x56, 0x62, - 0x23, 0x71, 0xc3, 0xa8, 0xdd, 0x5d, 0x8c, 0x8b, 0xe9, 0x8d, 0xee, 0x6a, 0x2f, 0xaf, 0x80, 0x78, - 0x1c, 0xc4, 0x03, 0x20, 0x71, 0xcf, 0x73, 0xf0, 0x14, 0x9c, 0x5a, 0xba, 0x7a, 0x99, 0xc5, 0x70, - 0xc1, 0xcd, 0x48, 0x55, 0x75, 0xce, 0x77, 0xce, 0xf9, 0xce, 0xd2, 0x55, 0x03, 0x5d, 0x3b, 0xa2, - 0x7b, 0x51, 0x1c, 0xb2, 0xd0, 0x6c, 0xb2, 0xcb, 0x88, 0x24, 0xd6, 0x2f, 0x35, 0xd8, 0x39, 0x88, - 0x89, 0xcd, 0xc8, 0x41, 0x18, 0x30, 0x9b, 0x06, 0x24, 0x7e, 0x4d, 0x7e, 0x4a, 0x49, 0xc2, 0x4c, - 0x80, 0x3a, 0x75, 0x47, 0xb5, 0x5b, 0xb5, 0x7b, 0x5d, 0x13, 0x17, 0x27, 0x69, 0xe0, 0x7a, 0xe4, - 0xd0, 0x66, 0xa7, 0xa3, 0xba, 0xd8, 0x1b, 0x40, 0x33, 0x61, 0x2e, 0x0d, 0x46, 0x0d, 0xb1, 0x1c, - 0x42, 0x0b, 0x97, 0x61, 0xca, 0x46, 0x46, 0x61, 0x4d, 0xe2, 0x78, 0xd4, 0x14, 0xeb, 0x75, 0x68, - 0x3b, 0x61, 0x90, 0x84, 0x1e, 0x19, 0xb5, 0x32, 0x4c, 0xe7, 0x94, 0x38, 0xb3, 0x28, 0xa4, 0x01, - 0x1b, 0xb5, 0xf9, 0x9e, 0x75, 0x17, 0xae, 0xcf, 0x79, 0x93, 0x44, 0xa8, 0x46, 0xcc, 0x1e, 0x34, - 0x22, 0xe5, 0xcf, 0xc0, 0x7a, 0x04, 0x83, 0x37, 0x74, 0x1a, 0xd8, 0xde, 0x22, 0x67, 0x95, 0x24, - 0xf7, 0x72, 0x20, 0xdc, 0x10, 0x92, 0xc2, 0xcd, 0x81, 0xb5, 0x01, 0xc3, 0x4c, 0x53, 0x02, 0x5b, - 0xbf, 0xd6, 0x60, 0xf3, 0x89, 0xeb, 0x1e, 0xc6, 0xa1, 0x43, 0x92, 0x64, 0x11, 0xe0, 0x06, 0x74, - 0x18, 0x89, 0x7d, 0xca, 0x51, 0x38, 0x6a, 0xc7, 0xdc, 0x05, 0x23, 0x4d, 0x48, 0x2c, 0x30, 0x7b, - 0xfb, 0xbd, 0x3d, 0x41, 0xe6, 0xde, 0x31, 0x6e, 0x99, 0x7d, 0x30, 0xec, 0x78, 0x9a, 0x20, 0x0b, - 0x0d, 0xe9, 0x0b, 0x09, 0xce, 0x90, 0x02, 0xb5, 0x70, 0xce, 0x5d, 0x15, 0xbe, 0xa6, 0xaf, 0x5d, - 0xa1, 0xaf, 0x53, 0xa1, 0xaf, 0x5b, 0xa5, 0x0f, 0x04, 0x55, 0x8f, 0xc0, 0x10, 0xf6, 0x10, 0x34, - 0xcd, 0x78, 0xe1, 0x8b, 0xa9, 0x0e, 0x7d, 0x07, 0x86, 0xb6, 0xeb, 0x52, 0x46, 0x43, 0x74, 0xfc, - 0x39, 0x75, 0x13, 0x74, 0xb7, 0x81, 0x14, 0xdc, 0x06, 0xb3, 0x18, 0xef, 0x22, 0x7e, 0x5f, 0xea, - 0x3c, 0xe8, 0x0c, 0x2d, 0x22, 0xe6, 0x9d, 0x52, 0x0a, 0xeb, 0x82, 0x8c, 0x4d, 0x45, 0x46, 0xae, - 0x69, 0x8d, 0x61, 0x34, 0x8f, 0xa6, 0xd8, 0x7f, 0x08, 0xd7, 0x9f, 0x12, 0x8f, 0x5c, 0x65, 0x09, - 0x59, 0x0d, 0x6c, 0x9f, 0xc8, 0xd2, 0xe3, 0x80, 0xf3, 0x4a, 0x0a, 0xf0, 0x6d, 0xd8, 0x7e, 0x49, - 0x13, 0xb6, 0x12, 0xce, 0xfa, 0x0e, 0x20, 0x17, 0xd0, 0xe0, 0xda, 0x14, 0xb9, 0xa0, 0x4c, 0x65, - 0x1a, 0x69, 0x61, 0x4e, 0x24, 0x12, 0xdd, 0x31, 0xb7, 0xa0, 0x97, 0x06, 0xf4, 0xe2, 0x4d, 0xe8, - 0xcc, 0x08, 0x4b, 0x44, 0xa1, 0x77, 0x44, 0x22, 0x4f, 0x89, 0xe7, 0x89, 0x3a, 0xef, 0x58, 0x9f, - 0xc3, 0x4e, 0xd5, 0xbe, 0x62, 0xf8, 0x2e, 0xf4, 0x72, 0xb6, 0x12, 0xb4, 0xd6, 0x58, 0x4c, 0xd7, - 0x10, 0xfa, 0x6f, 0x18, 0xb2, 0xa5, 0x1c, 0xb7, 0x6e, 0xc1, 0x50, 0xb7, 0x83, 0x38, 0x90, 0xc5, - 0x61, 0xb3, 0x34, 0x51, 0xe1, 0xcc, 0xa0, 0xad, 0xd2, 0x59, 0x4a, 0xe3, 0x7f, 0x52, 0xb8, 0x96, - 0x07, 0x5d, 0xed, 0xce, 0xf2, 0x1c, 0x55, 0x46, 0x86, 0x9c, 0x11, 0xb7, 0xa1, 0x1b, 0x49, 0x3f, - 0x89, 0xb4, 0xd3, 0xdb, 0x1f, 0x2a, 0x17, 0x32, 0xff, 0xf3, 0xd0, 0xc4, 0xd8, 0xc0, 0xfa, 0x68, - 0xbf, 0xb2, 0x9d, 0x53, 0x34, 0x56, 0xb5, 0xe5, 0x44, 0x28, 0xa4, 0x9b, 0xdc, 0x27, 0x7e, 0x18, - 0x5f, 0x0a, 0x3b, 0x86, 0xf5, 0x2d, 0x8e, 0x07, 0xc9, 0xa0, 0xa2, 0xfe, 0x0e, 0x16, 0x6a, 0xe6, - 0x73, 0xc6, 0xfc, 0x46, 0xc6, 0xbc, 0x0e, 0xe6, 0x26, 0xb4, 0x7d, 0x69, 0x4b, 0xd5, 0x72, 0xe6, - 0x9c, 0xf2, 0xc0, 0x7a, 0x0a, 0x3b, 0xc7, 0x91, 0x7b, 0xd5, 0xb0, 0xcc, 0x47, 0x4e, 0x3e, 0x82, - 0x64, 0x48, 0x82, 0x05, 0x6b, 0x17, 0xae, 0xcf, 0xa1, 0xa8, 0xe2, 0x5d, 0x87, 0xc1, 0xb3, 0x33, - 0x82, 0xd5, 0x91, 0xe5, 0xfe, 0xcf, 0x1a, 0x34, 0xc5, 0x0e, 0x8f, 0x98, 0x3b, 0xa3, 0x6c, 0x48, - 0x7b, 0xf5, 0x7c, 0x54, 0x68, 0xfc, 0x41, 0x85, 0x79, 0xa3, 0x38, 0x13, 0x9b, 0x95, 0x99, 0xd8, - 0x16, 0x6b, 0x8c, 0x5b, 0xa5, 0x45, 0x0c, 0x9f, 0xf9, 0xa4, 0x94, 0xe9, 0xeb, 0x2e, 0xa1, 0xaf, - 0x3c, 0x0d, 0x60, 0xd9, 0x34, 0xf8, 0xad, 0x06, 0xfd, 0xaf, 0x09, 0x3b, 0x0f, 0xe3, 0x19, 0x4f, - 0x52, 0x52, 0x69, 0x3f, 0xac, 0xd9, 0xf8, 0x62, 0x72, 0x72, 0xc9, 0x88, 0xcc, 0xae, 0xc1, 0xe3, - 0xc1, 0x9d, 0x43, 0x5b, 0x36, 0x9d, 0xc8, 0xb0, 0xb9, 0x09, 0xdd, 0xd7, 0x17, 0x13, 0x1c, 0x8f, - 0x61, 0x2c, 0xfb, 0x50, 0x88, 0xe1, 0x96, 0x1b, 0x87, 0x51, 0x44, 0x64, 0xa4, 0x06, 0x07, 0x3b, - 0xca, 0xc0, 0x5a, 0x99, 0x14, 0xee, 0x44, 0x0a, 0xac, 0x9d, 0x81, 0x1d, 0x69, 0xb0, 0x4e, 0x41, - 0x2c, 0x03, 0xeb, 0x8a, 0xaa, 0xf2, 0xa1, 0x73, 0x10, 0xa5, 0xc7, 0x89, 0x3d, 0x25, 0x7c, 0x12, - 0xb0, 0x90, 0xd9, 0xde, 0x24, 0xe5, 0x4b, 0xe1, 0xba, 0x61, 0x5e, 0x83, 0x7e, 0x44, 0x62, 0xac, - 0x4b, 0xb5, 0x5b, 0x47, 0xa2, 0x0c, 0xf3, 0x7f, 0xb0, 0x25, 0x96, 0x13, 0x1a, 0x4c, 0x66, 0x24, - 0x0e, 0x88, 0xe7, 0x87, 0x2e, 0x51, 0x71, 0xec, 0xc2, 0xa6, 0x3e, 0xe4, 0x8d, 0x29, 0x8e, 0x44, - 0x3c, 0xd6, 0x11, 0x0c, 0x8f, 0x4e, 0xf1, 0x5b, 0xcd, 0x3c, 0x1a, 0x4c, 0x9f, 0xda, 0xcc, 0xe6, - 0xdf, 0x00, 0xc4, 0xa7, 0xa1, 0x9b, 0x28, 0x83, 0xa8, 0xcd, 0xa4, 0x08, 0x71, 0x27, 0xd9, 0x91, - 0x24, 0x0d, 0x87, 0x7f, 0x7e, 0xc4, 0xa8, 0xaf, 0x0c, 0x5a, 0xdf, 0x8b, 0x20, 0x24, 0xf1, 0x16, - 0x74, 0x73, 0x67, 0x6b, 0x22, 0x5f, 0xeb, 0x59, 0xbe, 0xb2, 0x40, 0xf7, 0x60, 0x9d, 0x69, 0x2f, - 0x26, 0x58, 0xb5, 0xb6, 0xea, 0x8d, 0x6d, 0x25, 0x59, 0xf6, 0xd1, 0xfa, 0x0c, 0xe0, 0x95, 0x68, - 0x45, 0xe1, 0x31, 0xce, 0xc6, 0x22, 0x41, 0x48, 0xb4, 0x6f, 0x5f, 0x68, 0x76, 0xf8, 0x16, 0xc6, - 0xf4, 0x83, 0x4d, 0x3d, 0x07, 0x2b, 0x46, 0x3a, 0xf8, 0x57, 0x0d, 0x7a, 0x12, 0x41, 0x3a, 0x89, - 0x10, 0x0e, 0xb6, 0x5f, 0x06, 0x71, 0x2b, 0x43, 0x2c, 0x7f, 0x6d, 0x0a, 0x36, 0xb1, 0x0c, 0x93, - 0x73, 0x3b, 0x52, 0x56, 0x1a, 0xcb, 0xc4, 0xde, 0x85, 0xbe, 0xcc, 0x86, 0x12, 0x34, 0x96, 0x09, - 0xde, 0xe7, 0x1f, 0x6a, 0xf4, 0x44, 0xcc, 0xc2, 0xde, 0xfe, 0x8d, 0x92, 0x84, 0xf0, 0x71, 0x4f, - 0xfc, 0x3e, 0x0b, 0x58, 0x7c, 0x39, 0xbe, 0x0f, 0x90, 0xaf, 0x78, 0xdb, 0xcd, 0xc8, 0xa5, 0xaa, - 0x6c, 0x8c, 0xe4, 0xcc, 0xf6, 0x52, 0x15, 0xf9, 0xe3, 0xfa, 0xa3, 0x9a, 0xf5, 0x15, 0xac, 0x7f, - 0xe1, 0xcd, 0x68, 0x58, 0x50, 0x41, 0x29, 0xdf, 0xfe, 0x31, 0x8c, 0x55, 0xbc, 0x7c, 0x49, 0x03, - 0x5c, 0x4a, 0xba, 0xb0, 0xef, 0xc3, 0x48, 0x4d, 0x53, 0x8d, 0x27, 0xeb, 0xe5, 0xf7, 0x06, 0x40, - 0x0e, 0x66, 0x3e, 0x86, 0x31, 0x0d, 0x27, 0x58, 0x52, 0x67, 0xd4, 0x21, 0xb2, 0x05, 0x26, 0x31, - 0x71, 0xd2, 0x38, 0xa1, 0x67, 0x44, 0x8d, 0xc0, 0x1d, 0x15, 0x4b, 0xd5, 0x87, 0x0f, 0x61, 0x3b, - 0xd7, 0x75, 0x0b, 0x6a, 0xf5, 0x95, 0x6a, 0x0f, 0x61, 0x0b, 0xd5, 0x70, 0x70, 0xa5, 0x25, 0xa5, - 0xc6, 0x4a, 0xa5, 0x4f, 0x60, 0xb7, 0xe0, 0x27, 0xaf, 0xd4, 0x82, 0xaa, 0xb1, 0x52, 0xf5, 0x23, - 0xd8, 0x41, 0xd5, 0x73, 0x9b, 0xb2, 0xaa, 0x5e, 0xf3, 0x1f, 0xf8, 0xe9, 0x93, 0x78, 0x5a, 0xf2, - 0xb3, 0xb5, 0x52, 0xe9, 0x7d, 0xd8, 0x44, 0xa5, 0x8a, 0x9d, 0xf6, 0x55, 0x2a, 0x09, 0x71, 0x18, - 0x4e, 0x95, 0x82, 0x4a, 0x67, 0x95, 0x8a, 0xf5, 0x04, 0xfa, 0x2f, 0xd2, 0x29, 0x61, 0xde, 0x89, - 0xae, 0xfe, 0x7f, 0xdb, 0x40, 0x3f, 0xd7, 0xa1, 0x77, 0x30, 0x8d, 0xc3, 0x34, 0x2a, 0x75, 0xb9, - 0xac, 0xe1, 0xb9, 0x2e, 0x97, 0x32, 0xf7, 0xa0, 0x2f, 0x3f, 0xa0, 0x4a, 0x4c, 0x36, 0x97, 0x39, - 0x5f, 0xea, 0xfc, 0x12, 0x73, 0xc2, 0x7d, 0x56, 0x82, 0xe5, 0xf6, 0x2a, 0x94, 0xdf, 0xa7, 0x30, - 0x38, 0x95, 0x81, 0x28, 0x49, 0x99, 0xca, 0x3b, 0x99, 0xe5, 0xdc, 0xc1, 0xbd, 0x62, 0xc0, 0xb2, - 0x89, 0x5e, 0xc0, 0xe6, 0xdc, 0x66, 0xb9, 0x97, 0xac, 0x62, 0x2f, 0xf5, 0xf6, 0xb7, 0x14, 0x6c, - 0x51, 0x4b, 0x34, 0x58, 0x04, 0x4d, 0xe9, 0xcf, 0x7b, 0x30, 0x08, 0xe4, 0x47, 0x47, 0x33, 0xd1, - 0x28, 0x28, 0x96, 0x3e, 0x48, 0xc8, 0x86, 0x23, 0xfc, 0x5b, 0xc8, 0x46, 0x91, 0x5b, 0xcc, 0x07, - 0xaf, 0x08, 0x14, 0xf3, 0x23, 0x45, 0xff, 0x58, 0xde, 0xde, 0x16, 0x3d, 0x24, 0xf6, 0xff, 0x68, - 0x42, 0xe3, 0xc9, 0xe1, 0x97, 0xe6, 0x6b, 0x58, 0xaf, 0x3c, 0x73, 0xcc, 0x6c, 0xac, 0x2c, 0x7e, - 0x8c, 0x8d, 0xdf, 0x5a, 0x76, 0xac, 0x2e, 0x0e, 0x6b, 0x1c, 0xb3, 0x72, 0xab, 0xd0, 0x98, 0x8b, - 0xef, 0x2c, 0x1a, 0x73, 0xd9, 0x65, 0x64, 0xcd, 0xfc, 0x18, 0x5a, 0xf2, 0xb1, 0x64, 0x5e, 0x53, - 0xb2, 0xa5, 0x57, 0xd7, 0x78, 0xbb, 0xb2, 0xab, 0x15, 0x0f, 0x00, 0xf2, 0x27, 0x86, 0x39, 0x52, - 0x62, 0x73, 0xaf, 0xac, 0xf1, 0xee, 0x82, 0x13, 0x0d, 0x72, 0x0c, 0x1b, 0xd5, 0x67, 0x83, 0x59, - 0xe1, 0xa1, 0x7a, 0xc9, 0x1f, 0xdf, 0x5c, 0x7a, 0x5e, 0x84, 0xad, 0x3e, 0x1e, 0x34, 0xec, 0x92, - 0xa7, 0x88, 0x86, 0x5d, 0xfa, 0xea, 0x58, 0x33, 0xbf, 0x81, 0x61, 0xf9, 0xde, 0x6f, 0xfe, 0x5f, - 0x29, 0x2d, 0x7c, 0x8e, 0x8c, 0x6f, 0x2c, 0x39, 0xd5, 0x80, 0x1f, 0xc8, 0xd2, 0xc5, 0xbb, 0x46, - 0xc6, 0x72, 0xe1, 0x51, 0x30, 0xbe, 0x56, 0xde, 0xd4, 0x5a, 0x0f, 0xa0, 0x25, 0x6f, 0x90, 0x3a, - 0x65, 0xa5, 0x0b, 0xe5, 0xb8, 0x5f, 0xdc, 0xb5, 0xd6, 0x1e, 0xd4, 0x70, 0x4a, 0x75, 0x9e, 0x13, - 0x26, 0xeb, 0xb9, 0x68, 0x6a, 0x4e, 0x45, 0x6c, 0x72, 0x95, 0x93, 0x96, 0xf8, 0x0f, 0xe1, 0xe1, - 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xf1, 0xd0, 0x3c, 0x50, 0x10, 0x00, 0x00, + // 1440 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x72, 0xdc, 0x44, + 0x17, 0xf6, 0xec, 0x33, 0x67, 0x16, 0xdb, 0xf2, 0x92, 0xf1, 0xfc, 0x7f, 0x88, 0x11, 0x81, 0xa4, + 0xa8, 0x94, 0x2b, 0x38, 0x2c, 0x21, 0x5c, 0x40, 0x70, 0x52, 0x09, 0x54, 0x02, 0xae, 0xc4, 0xa6, + 0x8a, 0x1b, 0xa6, 0x64, 0xa9, 0x19, 0x8b, 0xd1, 0x48, 0x42, 0x6a, 0x79, 0x79, 0x05, 0x9e, 0x87, + 0xe2, 0x01, 0xa8, 0xe2, 0x9e, 0xe7, 0xe0, 0x29, 0x38, 0xdd, 0x7d, 0xd4, 0x5a, 0x66, 0x09, 0x5c, + 0x70, 0x33, 0x55, 0xdd, 0x7d, 0xce, 0x77, 0xbe, 0xb3, 0xaa, 0x7b, 0xa0, 0x63, 0x85, 0xee, 0x41, + 0x18, 0x05, 0x3c, 0x30, 0x1a, 0xfc, 0x3a, 0x64, 0xb1, 0xf9, 0x02, 0x76, 0x8f, 0x22, 0x66, 0x71, + 0x76, 0x14, 0xf8, 0xdc, 0x72, 0x7d, 0x16, 0xbd, 0x62, 0x3f, 0x27, 0x2c, 0xe6, 0x06, 0x40, 0xd5, + 0x75, 0x86, 0x95, 0xfd, 0xca, 0xdd, 0x8e, 0x81, 0x8b, 0xb3, 0xc4, 0x77, 0x3c, 0x76, 0x6c, 0xf1, + 0xf3, 0x61, 0x35, 0xdd, 0xb3, 0xcf, 0x99, 0x3d, 0x0d, 0x03, 0xd7, 0xe7, 0xc3, 0x9a, 0xd8, 0x33, + 0x9f, 0xc3, 0x8d, 0x39, 0xb4, 0x38, 0x0c, 0xfc, 0x98, 0x19, 0x7d, 0x68, 0xc4, 0xdc, 0x71, 0x7d, + 0x42, 0x1c, 0x40, 0x13, 0x97, 0x41, 0xc2, 0x09, 0x4d, 0xad, 0x59, 0x14, 0x11, 0xd2, 0x43, 0xe8, + 0xbf, 0x76, 0x27, 0xbe, 0xe5, 0x2d, 0xa2, 0xd3, 0x85, 0x5a, 0x88, 0x8b, 0x4c, 0x53, 0x4a, 0x4a, + 0xcd, 0xbe, 0xb9, 0x01, 0x83, 0x54, 0x53, 0x99, 0x36, 0x7f, 0xad, 0xc0, 0xe6, 0x63, 0xc7, 0x39, + 0x8e, 0x02, 0x9b, 0xc5, 0xf1, 0x22, 0xc0, 0x0d, 0x68, 0x73, 0x16, 0xcd, 0x5c, 0x81, 0x22, 0x50, + 0xdb, 0xc6, 0x1e, 0xd4, 0x93, 0x98, 0x29, 0x36, 0xdd, 0xc3, 0xee, 0x81, 0x8c, 0xd6, 0xc1, 0x29, + 0x6e, 0x19, 0x3d, 0xa8, 0x5b, 0xd1, 0x24, 0x1e, 0xd6, 0xf7, 0x6b, 0x8a, 0x0b, 0xf3, 0x2f, 0x86, + 0x8d, 0x74, 0x61, 0x5f, 0x3a, 0xc3, 0xa6, 0x04, 0xd5, 0x1e, 0xb7, 0x4a, 0x1e, 0xb7, 0x4b, 0x1e, + 0x77, 0xe4, 0x7a, 0x1d, 0x5a, 0x36, 0xd2, 0x0d, 0x3c, 0x36, 0x04, 0x0a, 0x41, 0x5d, 0xda, 0x43, + 0xd0, 0x84, 0x98, 0xf6, 0xc5, 0x62, 0x42, 0xae, 0xf7, 0x8d, 0x5d, 0x18, 0x58, 0x8e, 0xe3, 0x72, + 0x37, 0x40, 0xe2, 0xcf, 0x5c, 0x27, 0x46, 0xba, 0x35, 0x0c, 0xc1, 0x36, 0x18, 0x79, 0x7f, 0x29, + 0x0c, 0x2f, 0x74, 0x72, 0x74, 0xda, 0x16, 0xc5, 0xe2, 0xdd, 0x42, 0x5e, 0xab, 0xd2, 0xff, 0x4d, + 0xf2, 0x3f, 0xd3, 0x34, 0x47, 0x30, 0x9c, 0x47, 0x23, 0x4b, 0x0f, 0xe0, 0xc6, 0x13, 0xe6, 0xb1, + 0x37, 0x59, 0xc2, 0x40, 0xfa, 0xd6, 0x8c, 0xa9, 0x3c, 0x0a, 0xc0, 0x79, 0x25, 0x02, 0x7c, 0x07, + 0x76, 0x5e, 0xb8, 0x31, 0x5f, 0x09, 0x67, 0x7e, 0x0f, 0x90, 0x09, 0x68, 0x70, 0x6d, 0x8a, 0x5d, + 0xb9, 0x9c, 0x92, 0x8b, 0x41, 0xe4, 0x76, 0x28, 0x73, 0xdb, 0x36, 0xb6, 0xa0, 0x9b, 0xf8, 0xee, + 0xd5, 0xeb, 0xc0, 0x9e, 0x32, 0x2e, 0xb2, 0x2a, 0x36, 0x45, 0xee, 0xce, 0x99, 0xe7, 0x61, 0x5e, + 0x71, 0x69, 0x7e, 0x01, 0xbb, 0x65, 0xfb, 0x54, 0xd6, 0xef, 0x41, 0x37, 0x8b, 0x56, 0x8c, 0xd6, + 0x6a, 0x8b, 0xc3, 0x35, 0x80, 0xde, 0x6b, 0x8e, 0xd1, 0x22, 0xe2, 0xe6, 0x3e, 0x0c, 0x74, 0x8f, + 0xc8, 0x03, 0x55, 0x0f, 0x16, 0x4f, 0x62, 0x72, 0x67, 0x0a, 0x2d, 0xca, 0x60, 0x5a, 0xef, 0xff, + 0x5d, 0xad, 0x9a, 0x1e, 0x74, 0x34, 0x9d, 0xe5, 0x39, 0x2a, 0xcd, 0x01, 0xd9, 0xa9, 0xc6, 0xdb, + 0xd0, 0x09, 0x15, 0x4f, 0xa6, 0xec, 0x74, 0x0f, 0x07, 0x44, 0x21, 0xe5, 0x9f, 0xb9, 0xd6, 0x90, + 0xd6, 0xee, 0x40, 0xeb, 0xa5, 0x65, 0x9f, 0xa3, 0x31, 0x81, 0x6f, 0x87, 0xe4, 0x73, 0x5f, 0x08, + 0xce, 0xd8, 0x2c, 0x88, 0xae, 0xa5, 0xbd, 0xba, 0xf9, 0x1d, 0x4e, 0x01, 0x15, 0x35, 0x0a, 0xf7, + 0x6d, 0x2c, 0xce, 0x94, 0x67, 0x1a, 0xed, 0x8d, 0x34, 0xda, 0xda, 0x81, 0x5b, 0xd0, 0x9a, 0x29, + 0x7c, 0xaa, 0xdf, 0x94, 0x10, 0x59, 0x35, 0x9f, 0xc0, 0xee, 0x69, 0xe8, 0xbc, 0x69, 0xea, 0x65, + 0x93, 0xa5, 0x9a, 0xb2, 0x23, 0x37, 0xd4, 0x8c, 0xda, 0x83, 0x1b, 0x73, 0x28, 0x54, 0xb0, 0xeb, + 0xd0, 0x7f, 0x7a, 0xc1, 0xb0, 0x22, 0xd2, 0x7c, 0xff, 0x59, 0x81, 0x86, 0xdc, 0x11, 0x1e, 0x0b, + 0x32, 0x64, 0x43, 0xd9, 0xcb, 0xcd, 0x40, 0x8d, 0xdf, 0x2f, 0x45, 0xbb, 0x9e, 0x1f, 0x7d, 0x8d, + 0x12, 0xc1, 0x96, 0x54, 0x40, 0xbf, 0x29, 0x15, 0x72, 0xc6, 0xcc, 0x27, 0xa2, 0x18, 0xbe, 0xce, + 0x92, 0xf0, 0x15, 0x27, 0x00, 0x2c, 0x9b, 0x00, 0xbf, 0x55, 0xa0, 0xf7, 0x0d, 0xe3, 0x97, 0x41, + 0x34, 0x15, 0x49, 0x8a, 0x4b, 0x2d, 0x87, 0x75, 0x1a, 0x5d, 0x8d, 0xcf, 0xae, 0x39, 0x96, 0x85, + 0xcc, 0xa6, 0xf0, 0x07, 0x77, 0x8e, 0x2d, 0xd5, 0x68, 0x35, 0xb9, 0xb7, 0x09, 0x9d, 0x57, 0x57, + 0x63, 0x9c, 0x82, 0x41, 0xa4, 0x7a, 0x4f, 0x8a, 0xe1, 0x96, 0x13, 0x05, 0x61, 0xc8, 0x94, 0xa7, + 0x75, 0x01, 0x76, 0x92, 0x82, 0x35, 0x53, 0x29, 0xdc, 0x09, 0x09, 0xac, 0x95, 0x82, 0x9d, 0x68, + 0xb0, 0x76, 0x4e, 0x2c, 0x05, 0xeb, 0xc8, 0xaa, 0x9a, 0x41, 0xfb, 0x28, 0x4c, 0x4e, 0x63, 0x6b, + 0xc2, 0x44, 0xf7, 0xf3, 0x80, 0x5b, 0xde, 0x38, 0x11, 0x4b, 0x49, 0xbd, 0x6e, 0x6c, 0x43, 0x2f, + 0x64, 0x11, 0xd6, 0x25, 0xed, 0x56, 0x31, 0x50, 0x75, 0xe3, 0x7f, 0xb0, 0x25, 0x97, 0x63, 0xd7, + 0x1f, 0x4f, 0x59, 0xe4, 0x33, 0x6f, 0x16, 0x38, 0x8c, 0xfc, 0xd8, 0x83, 0x4d, 0x7d, 0x28, 0x9a, + 0x51, 0x1e, 0x49, 0x7f, 0xcc, 0x13, 0x18, 0x9c, 0x9c, 0xe3, 0x37, 0x97, 0x7b, 0xae, 0x3f, 0x79, + 0x62, 0x71, 0x4b, 0x8c, 0x7a, 0xc4, 0x77, 0x03, 0x27, 0x26, 0x83, 0xa8, 0xcd, 0x95, 0x08, 0x73, + 0xc6, 0xe9, 0x91, 0x0a, 0x1a, 0xce, 0xf8, 0xec, 0x88, 0xbb, 0x33, 0x32, 0x68, 0xfe, 0x20, 0x9d, + 0x50, 0x81, 0x37, 0xa1, 0x93, 0x91, 0xad, 0xc8, 0x7c, 0xad, 0xa7, 0xf9, 0x4a, 0x1d, 0x3d, 0x80, + 0x75, 0xae, 0x59, 0x8c, 0xb1, 0x6a, 0x2d, 0xea, 0x8d, 0x1d, 0x92, 0x2c, 0x72, 0x34, 0x3f, 0x07, + 0x78, 0x29, 0x5b, 0x51, 0x32, 0xc6, 0x79, 0x98, 0x0f, 0x10, 0x06, 0x7a, 0x66, 0x5d, 0xe9, 0xe8, + 0x88, 0x2d, 0xf4, 0xe9, 0x47, 0xcb, 0xf5, 0x6c, 0xba, 0x0b, 0xd4, 0xcd, 0xbf, 0x2a, 0xd0, 0x55, + 0x08, 0x8a, 0x24, 0x42, 0xd8, 0xd8, 0x7e, 0x29, 0xc4, 0x7e, 0x8a, 0x58, 0xfc, 0xc2, 0xe4, 0x6c, + 0x62, 0x19, 0xc6, 0x97, 0x56, 0x48, 0x56, 0x6a, 0xcb, 0xc4, 0xee, 0x40, 0x4f, 0x65, 0x83, 0x04, + 0xeb, 0xcb, 0x04, 0xef, 0x89, 0xef, 0x31, 0x32, 0x91, 0xf3, 0xaf, 0x7b, 0x78, 0xb3, 0x20, 0x21, + 0x39, 0x1e, 0xc8, 0xdf, 0xa7, 0x3e, 0x8f, 0xae, 0x47, 0xf7, 0x00, 0xb2, 0x95, 0x68, 0xbb, 0x29, + 0xbb, 0xa6, 0xca, 0x46, 0x4f, 0x2e, 0x2c, 0x2f, 0x21, 0xcf, 0x1f, 0x55, 0x1f, 0x56, 0xcc, 0xaf, + 0x61, 0xfd, 0x4b, 0x6f, 0xea, 0x06, 0x39, 0x15, 0x94, 0x9a, 0x59, 0x3f, 0x05, 0x11, 0xf9, 0x2b, + 0x96, 0xae, 0x8f, 0x4b, 0x15, 0x2e, 0xec, 0xfb, 0x20, 0xa4, 0x09, 0xaa, 0xf1, 0x54, 0xbd, 0xfc, + 0x5e, 0x03, 0xc8, 0xc0, 0x8c, 0x47, 0x30, 0x72, 0x83, 0x31, 0x96, 0xd4, 0x85, 0x6b, 0x33, 0xd5, + 0x02, 0xe3, 0x88, 0xd9, 0x49, 0x14, 0xbb, 0x17, 0x8c, 0x46, 0xe0, 0x2e, 0xf9, 0x52, 0xe6, 0xf0, + 0x11, 0xec, 0x64, 0xba, 0x4e, 0x4e, 0xad, 0xba, 0x52, 0xed, 0x01, 0x6c, 0xa1, 0x1a, 0x0e, 0xae, + 0xa4, 0xa0, 0x54, 0x5b, 0xa9, 0xf4, 0x29, 0xec, 0xe5, 0x78, 0x8a, 0x4a, 0xcd, 0xa9, 0xd6, 0x57, + 0xaa, 0x7e, 0x0c, 0xbb, 0xa8, 0x7a, 0x69, 0xb9, 0xbc, 0xac, 0xd7, 0xf8, 0x07, 0x3c, 0x67, 0x2c, + 0x9a, 0x14, 0x78, 0x36, 0x57, 0x2a, 0x7d, 0x00, 0x9b, 0xa8, 0x54, 0xb2, 0xd3, 0x7a, 0x93, 0x4a, + 0xcc, 0x6c, 0x8e, 0x53, 0x25, 0xa7, 0xd2, 0x5e, 0xa5, 0x62, 0x3e, 0x86, 0xde, 0xf3, 0x64, 0xc2, + 0xb8, 0x77, 0xa6, 0xab, 0xff, 0xdf, 0x36, 0xd0, 0x2f, 0x55, 0xe8, 0x1e, 0x4d, 0xa2, 0x20, 0x09, + 0x0b, 0x5d, 0xae, 0x6a, 0x78, 0xae, 0xcb, 0x95, 0xcc, 0x5d, 0xe8, 0xa9, 0x0f, 0x28, 0x89, 0xa9, + 0xe6, 0x32, 0xe6, 0x4b, 0x5d, 0x5c, 0x5c, 0xce, 0x04, 0x67, 0x12, 0x2c, 0xb6, 0x57, 0xae, 0xfc, + 0x3e, 0x83, 0xfe, 0xb9, 0x72, 0x84, 0x24, 0x55, 0x2a, 0x6f, 0xa7, 0x96, 0x33, 0x82, 0x07, 0x79, + 0x87, 0x55, 0x13, 0x3d, 0x87, 0xcd, 0xb9, 0xcd, 0x62, 0x2f, 0x99, 0xf9, 0x5e, 0xea, 0x1e, 0x6e, + 0x11, 0x6c, 0x5e, 0x4b, 0x36, 0x58, 0x08, 0x0d, 0xc5, 0xe7, 0x7d, 0xe8, 0xfb, 0xea, 0xa3, 0xa3, + 0x23, 0x51, 0xcb, 0x29, 0x16, 0x3e, 0x48, 0x18, 0x0d, 0x5b, 0xf2, 0x5b, 0x18, 0x8d, 0x7c, 0x6c, + 0x31, 0x1f, 0xa2, 0x22, 0x50, 0x6c, 0x16, 0x52, 0xf8, 0x47, 0xea, 0xc6, 0xb6, 0xe8, 0xbd, 0x70, + 0xf8, 0x47, 0x03, 0x6a, 0x8f, 0x8f, 0xbf, 0x32, 0x5e, 0xc1, 0x7a, 0xe9, 0xbd, 0x63, 0xa4, 0x63, + 0x65, 0xf1, 0xab, 0x6a, 0xf4, 0xd6, 0xb2, 0x63, 0xba, 0x38, 0xac, 0x09, 0xcc, 0xd2, 0xad, 0x42, + 0x63, 0x2e, 0xbe, 0xb3, 0x68, 0xcc, 0x65, 0x97, 0x91, 0x35, 0xe3, 0x13, 0x68, 0xaa, 0x37, 0x91, + 0xb1, 0x4d, 0xb2, 0x85, 0xc7, 0xd5, 0x68, 0xa7, 0xb4, 0xab, 0x15, 0x8f, 0x00, 0xb2, 0x97, 0x84, + 0x31, 0x24, 0xb1, 0xb9, 0xc7, 0xd4, 0x68, 0x6f, 0xc1, 0x89, 0x06, 0x39, 0x85, 0x8d, 0xf2, 0x53, + 0xc1, 0x28, 0xc5, 0xa1, 0x7c, 0xb1, 0x1f, 0xdd, 0x5a, 0x7a, 0x9e, 0x87, 0x2d, 0x3f, 0x18, 0x34, + 0xec, 0x92, 0xe7, 0x87, 0x86, 0x5d, 0xfa, 0xd2, 0x58, 0x33, 0xbe, 0x85, 0x41, 0xf1, 0xae, 0x6f, + 0xfc, 0x9f, 0x94, 0x16, 0x3e, 0x41, 0x46, 0x37, 0x97, 0x9c, 0x6a, 0xc0, 0x0f, 0x55, 0xe9, 0xe2, + 0x5d, 0x23, 0x8d, 0x72, 0xee, 0x21, 0x30, 0xda, 0x2e, 0x6e, 0x6a, 0xad, 0xfb, 0xd0, 0x54, 0x37, + 0x48, 0x9d, 0xb2, 0xc2, 0x85, 0x72, 0xd4, 0xcb, 0xef, 0x9a, 0x6b, 0xf7, 0x2b, 0x38, 0xa5, 0xda, + 0xcf, 0x18, 0x57, 0xf5, 0x9c, 0x37, 0x35, 0xa7, 0x22, 0x37, 0x85, 0xca, 0x59, 0x53, 0xfe, 0x17, + 0xf0, 0xe0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x15, 0x52, 0xba, 0x18, 0x10, 0x00, 0x00, } diff --git a/api/grpc/types/api.proto b/api/grpc/types/api.proto index 08963cc7a..641dfef06 100644 --- a/api/grpc/types/api.proto +++ b/api/grpc/types/api.proto @@ -18,20 +18,19 @@ service API { message CreateContainerRequest { string id = 1; // ID of container string bundlePath = 2; // path to OCI bundle - string stdin = 3; // path to the file where stdin will be read (optional) - string stdout = 4; // path to file where stdout will be written (optional) - string stderr = 5; // path to file where stderr will be written (optional) - string console = 6; // path to the console for a container (optional) string checkpoint = 7; // checkpoint name if you want to create immediate checkpoint (optional) } message CreateContainerResponse { uint32 pid = 1; // PID of the containers main process + string stdin = 1; // path to the file where stdin will be read (optional) + string stdout = 2; // path to file where stdout will be written (optional) + string stderr = 3; // path to file where stderr will be written (optional) } message SignalRequest { string id = 1; // ID of container - uint32 pid = 2; // PID of process inside container + string pid = 2; // PID of process inside container uint32 signal = 3; // Signal which will be sent, you can find value in "man 7 signal" } @@ -58,7 +57,6 @@ message User { } message AddProcessResponse { - uint32 pid = 1; // PID of process is returned in case of success } message CreateCheckpointRequest { @@ -101,7 +99,7 @@ message ContainerState { } message Process { - uint32 pid = 1; + string pid = 1; bool terminal = 2; // Use tty for container stdio User user = 3; // User under which process will be run repeated string args = 4; // Arguments for process, first is binary path itself @@ -119,9 +117,8 @@ message Container { // Machine is information about machine on which containerd is run message Machine { - string id = 1; // ID of machine - uint32 cpus = 2; // number of cpus - uint64 memory = 3; // amount of memory + uint32 cpus = 1; // number of cpus + uint64 memory = 2; // amount of memory } // StateResponse is information about containerd daemon @@ -147,7 +144,7 @@ message Event { string id = 2; uint32 status = 3; string bundlePath = 4; - uint32 pid = 5; + string pid = 5; uint32 signal = 7; Process process = 8; repeated Container containers = 9; diff --git a/containerd-shim/main.go b/containerd-shim/main.go new file mode 100644 index 000000000..0de23bc80 --- /dev/null +++ b/containerd-shim/main.go @@ -0,0 +1,210 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/docker/containerd/util" + "github.com/opencontainers/runc/libcontainer" + "github.com/opencontainers/specs" +) + +const ( + bufferSize = 2048 +) + +type stdio struct { + stdin *os.File + stdout *os.File + stderr *os.File + console string +} + +func (s *stdio) Close() error { + err := s.stdin.Close() + if oerr := s.stdout.Close(); err == nil { + err = oerr + } + if oerr := s.stderr.Close(); err == nil { + err = oerr + } + return err +} + +// containerd-shim is a small shim that sits in front of a runc implementation +// that allows it to be repartented to init and handle reattach from the caller. +// +// the cwd of the shim should be the bundle for the container. Arg1 should be the path +// to the state directory where the shim can locate fifos and other information. +// +// └── shim +// ├── control +// ├── stderr +// ├── stdin +// ├── stdout +// ├── pid +// └── exit +func main() { + if len(os.Args) < 2 { + logrus.Fatal("shim: no arguments provided") + } + // start handling signals as soon as possible so that things are properly reaped + // or if runc exits before we hit the handler + signals := make(chan os.Signal, bufferSize) + signal.Notify(signals) + // set the shim as the subreaper for all orphaned processes created by the container + if err := util.SetSubreaper(1); err != nil { + logrus.WithField("error", err).Fatal("shim: set as subreaper") + } + // open the exit pipe + f, err := os.OpenFile(filepath.Join(os.Args[1], "exit"), syscall.O_WRONLY, 0) + if err != nil { + logrus.WithField("error", err).Fatal("shim: open exit pipe") + } + defer f.Close() + // open the fifos for use with the command + std, err := openContainerSTDIO(os.Args[1]) + if err != nil { + logrus.WithField("error", err).Fatal("shim: open container STDIO from fifo") + } + // star the container process by invoking runc + runcPid, err := startRunc(std, os.Args[2]) + if err != nil { + logrus.WithField("error", err).Fatal("shim: start runc") + } + var exitShim bool + for s := range signals { + logrus.WithField("signal", s).Debug("shim: received signal") + switch s { + case syscall.SIGCHLD: + exits, err := util.Reap() + if err != nil { + logrus.WithField("error", err).Error("shim: reaping child processes") + } + for _, e := range exits { + // check to see if runc is one of the processes that has exited + if e.Pid == runcPid { + exitShim = true + logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).Info("shim: runc exited") + + if err := writeInt(filepath.Join(os.Args[1], "exitStatus"), e.Status); err != nil { + logrus.WithFields(logrus.Fields{"error": err, "status": e.Status}).Error("shim: write exit status") + } + } + } + } + // runc has exited so the shim can also exit + if exitShim { + if err := std.Close(); err != nil { + logrus.WithField("error", err).Error("shim: close stdio") + } + if err := deleteContainer(os.Args[2]); err != nil { + logrus.WithField("error", err).Error("shim: delete runc state") + } + return + } + } +} + +// startRunc starts runc detached and returns the container's pid +func startRunc(s *stdio, id string) (int, error) { + pidFile := filepath.Join(os.Args[1], "pid") + cmd := exec.Command("runc", "--id", id, "start", "-d", "--console", s.console, "--pid-file", pidFile) + cmd.Stdin = s.stdin + cmd.Stdout = s.stdout + cmd.Stderr = s.stderr + // set the parent death signal to SIGKILL so that if the shim dies the container + // process also dies + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + if err := cmd.Run(); err != nil { + return -1, err + } + data, err := ioutil.ReadFile(pidFile) + if err != nil { + return -1, err + } + return strconv.Atoi(string(data)) +} + +func deleteContainer(id string) error { + return exec.Command("runc", "--id", id, "delete").Run() +} + +// openContainerSTDIO opens the pre-created fifo's for use with the container +// in RDWR so that they remain open if the other side stops listening +func openContainerSTDIO(dir string) (*stdio, error) { + s := &stdio{} + spec, err := getSpec() + if err != nil { + return nil, err + } + if spec.Process.Terminal { + console, err := libcontainer.NewConsole(int(spec.Process.User.UID), int(spec.Process.User.GID)) + if err != nil { + return nil, err + } + s.console = console.Path() + stdin, err := os.OpenFile(filepath.Join(dir, "stdin"), syscall.O_RDWR, 0) + if err != nil { + return nil, err + } + go func() { + io.Copy(console, stdin) + }() + stdout, err := os.OpenFile(filepath.Join(dir, "stdout"), syscall.O_RDWR, 0) + if err != nil { + return nil, err + } + go func() { + io.Copy(stdout, console) + console.Close() + }() + return s, nil + } + for name, dest := range map[string]**os.File{ + "stdin": &s.stdin, + "stdout": &s.stdout, + "stderr": &s.stderr, + } { + f, err := os.OpenFile(filepath.Join(dir, name), syscall.O_RDWR, 0) + if err != nil { + return nil, err + } + *dest = f + } + return s, nil +} + +func getSpec() (*specs.Spec, error) { + var s specs.Spec + f, err := os.Open("config.json") + if err != nil { + return nil, err + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&s); err != nil { + return nil, err + } + return &s, nil +} + +func writeInt(path string, i int) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + _, err = fmt.Fprintf(f, "%d", i) + return err +} diff --git a/containerd/main.go b/containerd/main.go index d1abce87a..d56da7129 100644 --- a/containerd/main.go +++ b/containerd/main.go @@ -4,6 +4,7 @@ import ( "log" "net" "os" + "os/signal" "runtime" "sync" "syscall" @@ -29,11 +30,6 @@ const ( ) var daemonFlags = []cli.Flag{ - cli.StringFlag{ - Name: "id", - Value: getDefaultID(), - Usage: "unique containerd id to identify the instance", - }, cli.BoolFlag{ Name: "debug", Usage: "enable debug output in the logs", @@ -88,7 +84,6 @@ func main() { } app.Action = func(context *cli.Context) { if err := daemon( - context.String("id"), context.String("listen"), context.String("state-dir"), context.Int("concurrency"), @@ -172,9 +167,12 @@ func processMetrics() { }() } -func daemon(id, address, stateDir string, concurrency int, oom bool) error { +func daemon(address, stateDir string, concurrency int, oom bool) error { + // setup a standard reaper so that we don't leave any zombies if we are still alive + // this is just good practice because we are spawning new processes + go reapProcesses() tasks := make(chan *supervisor.StartTask, concurrency*100) - sv, err := supervisor.New(id, stateDir, tasks, oom) + sv, err := supervisor.New(stateDir, tasks, oom) if err != nil { return err } @@ -184,17 +182,6 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error { w := supervisor.NewWorker(sv, wg) go w.Start() } - // only set containerd as the subreaper if it is not an init process - if pid := os.Getpid(); pid != 1 { - logrus.WithFields(logrus.Fields{ - "pid": pid, - }).Debug("containerd is not init, set as subreaper") - if err := setSubReaper(); err != nil { - return err - } - } - // start the signal handler in the background. - go startSignalHandler(sv) if err := sv.Start(); err != nil { return err } @@ -211,6 +198,19 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error { return s.Serve(l) } +func reapProcesses() { + s := make(chan os.Signal, 2048) + signal.Notify(s, syscall.SIGCHLD) + if err := util.SetSubreaper(1); err != nil { + logrus.WithField("error", err).Error("containerd: set subpreaper") + } + for range s { + if _, err := util.Reap(); err != nil { + logrus.WithField("error", err).Error("containerd: reap child processes") + } + } +} + // getDefaultID returns the hostname for the instance host func getDefaultID() string { hostname, err := os.Hostname() diff --git a/containerd/reap_linux.go b/containerd/reap_linux.go deleted file mode 100644 index 0459c46b0..000000000 --- a/containerd/reap_linux.go +++ /dev/null @@ -1,67 +0,0 @@ -// +build linux - -package main - -import ( - "os" - "os/signal" - "syscall" - - "github.com/Sirupsen/logrus" - "github.com/docker/containerd/supervisor" - "github.com/docker/containerd/util" - "github.com/opencontainers/runc/libcontainer/utils" -) - -const signalBufferSize = 2048 - -func startSignalHandler(supervisor *supervisor.Supervisor) { - logrus.WithFields(logrus.Fields{ - "bufferSize": signalBufferSize, - }).Debug("containerd: starting signal handler") - signals := make(chan os.Signal, signalBufferSize) - signal.Notify(signals) - for s := range signals { - switch s { - case syscall.SIGTERM, syscall.SIGINT: - supervisor.Stop(signals) - case syscall.SIGCHLD: - exits, err := reap() - if err != nil { - logrus.WithField("error", err).Error("containerd: reaping child processes") - } - for _, e := range exits { - supervisor.SendEvent(e) - } - } - } - supervisor.Close() - os.Exit(0) -} - -func reap() (exits []*supervisor.Event, err error) { - var ( - ws syscall.WaitStatus - rus syscall.Rusage - ) - for { - pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus) - if err != nil { - if err == syscall.ECHILD { - return exits, nil - } - return exits, err - } - if pid <= 0 { - return exits, nil - } - e := supervisor.NewEvent(supervisor.ExitEventType) - e.Pid = pid - e.Status = utils.ExitStatus(ws) - exits = append(exits, e) - } -} - -func setSubReaper() error { - return util.SetSubreaper(1) -} diff --git a/ctr/container.go b/ctr/container.go index af793c4e4..473101377 100644 --- a/ctr/container.go +++ b/ctr/container.go @@ -4,10 +4,10 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net" "os" "path/filepath" + "strings" "syscall" "text/tabwriter" "time" @@ -15,7 +15,6 @@ import ( "github.com/codegangsta/cli" "github.com/docker/containerd/api/grpc/types" "github.com/docker/docker/pkg/term" - "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/specs" netcontext "golang.org/x/net/context" "google.golang.org/grpc" @@ -45,6 +44,7 @@ var containersCommand = cli.Command{ listCommand, startCommand, statsCommand, + attachCommand, }, Action: listContainers, } @@ -62,15 +62,97 @@ func listContainers(context *cli.Context) { fatal(err.Error(), 1) } w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) - fmt.Fprint(w, "ID\tPATH\tSTATUS\tPID1\n") + fmt.Fprint(w, "ID\tPATH\tSTATUS\tPROCESSES\n") for _, c := range resp.Containers { - fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", c.Id, c.BundlePath, c.Status, c.Processes[0].Pid) + procs := []string{} + for _, p := range c.Processes { + procs = append(procs, p.Pid) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Id, c.BundlePath, c.Status, strings.Join(procs, ",")) } if err := w.Flush(); err != nil { fatal(err.Error(), 1) } } +var attachCommand = cli.Command{ + Name: "attach", + Usage: "attach to a running container", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "state-dir", + Value: "/run/containerd", + Usage: "runtime state directory", + }, + cli.StringFlag{ + Name: "pid,p", + Value: "init", + Usage: "specify the process id to attach to", + }, + }, + Action: func(context *cli.Context) { + var ( + id = context.Args().First() + pid = context.String("pid") + ) + if id == "" { + fatal("container id cannot be empty", 1) + } + c := getClient(context) + events, err := c.Events(netcontext.Background(), &types.EventsRequest{}) + if err != nil { + fatal(err.Error(), 1) + } + type bundleState struct { + Bundle string `json:"bundle"` + } + f, err := os.Open(filepath.Join(context.String("state-dir"), id, "state.json")) + if err != nil { + fatal(err.Error(), 1) + } + var s bundleState + err = json.NewDecoder(f).Decode(&s) + f.Close() + if err != nil { + fatal(err.Error(), 1) + } + mkterm, err := readTermSetting(s.Bundle) + if err != nil { + fatal(err.Error(), 1) + } + if mkterm { + s, err := term.SetRawTerminal(os.Stdin.Fd()) + if err != nil { + fatal(err.Error(), 1) + } + state = s + } + if err := attachStdio( + filepath.Join(context.String("state-dir"), id, pid, "stdin"), + filepath.Join(context.String("state-dir"), id, pid, "stdout"), + filepath.Join(context.String("state-dir"), id, pid, "stderr"), + ); err != nil { + fatal(err.Error(), 1) + } + go func() { + io.Copy(stdin, os.Stdin) + if state != nil { + term.RestoreTerminal(os.Stdin.Fd(), state) + } + stdin.Close() + }() + for { + e, err := events.Recv() + if err != nil { + fatal(err.Error(), 1) + } + if e.Id == id && e.Type == "exit" && e.Pid == pid { + os.Exit(int(e.Status)) + } + } + }, +} + var startCommand = cli.Command{ Name: "start", Usage: "start a container", @@ -110,24 +192,25 @@ var startCommand = cli.Command{ BundlePath: bpath, Checkpoint: context.String("checkpoint"), } + resp, err := c.CreateContainer(netcontext.Background(), r) + if err != nil { + fatal(err.Error(), 1) + } if context.Bool("attach") { mkterm, err := readTermSetting(bpath) if err != nil { fatal(err.Error(), 1) } if mkterm { - if err := attachTty(&r.Console); err != nil { - fatal(err.Error(), 1) - } - } else { - if err := attachStdio(&r.Stdin, &r.Stdout, &r.Stderr); err != nil { + s, err := term.SetRawTerminal(os.Stdin.Fd()) + if err != nil { fatal(err.Error(), 1) } + state = s + } + if err := attachStdio(resp.Stdin, resp.Stdout, resp.Stderr); err != nil { + fatal(err.Error(), 1) } - } - resp, err := c.CreateContainer(netcontext.Background(), r) - if err != nil { - fatal(err.Error(), 1) } if context.Bool("attach") { restoreAndCloseStdin := func() { @@ -146,13 +229,11 @@ var startCommand = cli.Command{ restoreAndCloseStdin() fatal(err.Error(), 1) } - if e.Id == id && e.Type == "exit" { + if e.Id == id && e.Type == "exit" && e.Pid == "init" { restoreAndCloseStdin() os.Exit(int(e.Status)) } } - } else { - fmt.Println(resp.Pid) } }, } @@ -177,69 +258,24 @@ func readTermSetting(path string) (bool, error) { return spec.Process.Terminal, nil } -func attachTty(consolePath *string) error { - console, err := libcontainer.NewConsole(os.Getuid(), os.Getgid()) +func attachStdio(stdins, stdout, stderr string) error { + stdinf, err := os.OpenFile(stdins, syscall.O_RDWR, 0) if err != nil { return err } - *consolePath = console.Path() - stdin = console - go func() { - io.Copy(os.Stdout, console) - console.Close() - }() - s, err := term.SetRawTerminal(os.Stdin.Fd()) - if err != nil { - return err - } - state = s - return nil -} + stdin = stdinf -func attachStdio(stdins, stdout, stderr *string) error { - dir, err := ioutil.TempDir("", "ctr-") + stdoutf, err := os.OpenFile(stdout, syscall.O_RDWR, 0) if err != nil { return err } - for _, p := range []struct { - path string - flag int - done func(f *os.File) - }{ - { - path: filepath.Join(dir, "stdin"), - flag: syscall.O_RDWR, - done: func(f *os.File) { - *stdins = filepath.Join(dir, "stdin") - stdin = f - }, - }, - { - path: filepath.Join(dir, "stdout"), - flag: syscall.O_RDWR, - done: func(f *os.File) { - *stdout = filepath.Join(dir, "stdout") - go io.Copy(os.Stdout, f) - }, - }, - { - path: filepath.Join(dir, "stderr"), - flag: syscall.O_RDWR, - done: func(f *os.File) { - *stderr = filepath.Join(dir, "stderr") - go io.Copy(os.Stderr, f) - }, - }, - } { - if err := syscall.Mkfifo(p.path, 0755); err != nil { - return fmt.Errorf("mkfifo: %s %v", p.path, err) - } - f, err := os.OpenFile(p.path, p.flag, 0) - if err != nil { - return fmt.Errorf("open: %s %v", p.path, err) - } - p.done(f) + go io.Copy(os.Stdout, stdoutf) + + stderrf, err := os.OpenFile(stderr, syscall.O_RDWR, 0) + if err != nil { + return err } + go io.Copy(os.Stderr, stderrf) return nil } @@ -247,7 +283,7 @@ var killCommand = cli.Command{ Name: "kill", Usage: "send a signal to a container or its processes", Flags: []cli.Flag{ - cli.IntFlag{ + cli.StringFlag{ Name: "pid,p", Usage: "pid of the process to signal within the container", }, @@ -265,7 +301,7 @@ var killCommand = cli.Command{ c := getClient(context) if _, err := c.Signal(netcontext.Background(), &types.SignalRequest{ Id: id, - Pid: uint32(context.Int("pid")), + Pid: context.String("pid"), Signal: uint32(context.Int("signal")), }); err != nil { fatal(err.Error(), 1) @@ -308,55 +344,58 @@ var execCommand = cli.Command{ }, }, Action: func(context *cli.Context) { - p := &types.AddProcessRequest{ - Args: context.Args(), - Cwd: context.String("cwd"), - Terminal: context.Bool("tty"), - Id: context.String("id"), - Env: context.StringSlice("env"), - User: &types.User{ - Uid: uint32(context.Int("uid")), - Gid: uint32(context.Int("gid")), - }, - } - c := getClient(context) - events, err := c.Events(netcontext.Background(), &types.EventsRequest{}) - if err != nil { - fatal(err.Error(), 1) - } - if context.Bool("attach") { - if p.Terminal { - if err := attachTty(&p.Console); err != nil { - fatal(err.Error(), 1) - } - } else { - if err := attachStdio(&p.Stdin, &p.Stdout, &p.Stderr); err != nil { - fatal(err.Error(), 1) - } + panic("not implemented") + /* + p := &types.AddProcessRequest{ + Args: context.Args(), + Cwd: context.String("cwd"), + Terminal: context.Bool("tty"), + Id: context.String("id"), + Env: context.StringSlice("env"), + User: &types.User{ + Uid: uint32(context.Int("uid")), + Gid: uint32(context.Int("gid")), + }, } - } - r, err := c.AddProcess(netcontext.Background(), p) - if err != nil { - fatal(err.Error(), 1) - } - if context.Bool("attach") { - go func() { - io.Copy(stdin, os.Stdin) - if state != nil { - term.RestoreTerminal(os.Stdin.Fd(), state) - } - stdin.Close() - }() - for { - e, err := events.Recv() + c := getClient(context) + events, err := c.Events(netcontext.Background(), &types.EventsRequest{}) + if err != nil { + fatal(err.Error(), 1) + } + if context.Bool("attach") { + if p.Terminal { + if err := attachTty(&p.Console); err != nil { + fatal(err.Error(), 1) + } + } else { + if err := attachStdio(&p.Stdin, &p.Stdout, &p.Stderr); err != nil { + fatal(err.Error(), 1) + } + } + } + r, err := c.AddProcess(netcontext.Background(), p) if err != nil { fatal(err.Error(), 1) } - if e.Pid == r.Pid && e.Type == "exit" { - os.Exit(int(e.Status)) + if context.Bool("attach") { + go func() { + io.Copy(stdin, os.Stdin) + if state != nil { + term.RestoreTerminal(os.Stdin.Fd(), state) + } + stdin.Close() + }() + for { + e, err := events.Recv() + if err != nil { + fatal(err.Error(), 1) + } + if e.Pid == r.Pid && e.Type == "exit" { + os.Exit(int(e.Status)) + } + } } - } - } + */ }, } diff --git a/ctr/events.go b/ctr/events.go index 71cdcf22c..93d01d61f 100644 --- a/ctr/events.go +++ b/ctr/events.go @@ -27,7 +27,7 @@ var eventsCommand = cli.Command{ if err != nil { fatal(err.Error(), 1) } - fmt.Fprintf(w, "%s\t%s\t%d\t%d\n", e.Type, e.Id, e.Pid, e.Status) + fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", e.Type, e.Id, e.Pid, e.Status) w.Flush() } }, diff --git a/linux/linux.go b/linux/linux.go deleted file mode 100644 index 0e8479821..000000000 --- a/linux/linux.go +++ /dev/null @@ -1,955 +0,0 @@ -// +build libcontainer - -package linux - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - goruntime "runtime" - "strconv" - "strings" - "syscall" - "time" - - "github.com/docker/containerd/runtime" - "github.com/opencontainers/runc/libcontainer" - "github.com/opencontainers/runc/libcontainer/configs" - _ "github.com/opencontainers/runc/libcontainer/nsenter" - "github.com/opencontainers/runc/libcontainer/seccomp" - "github.com/opencontainers/specs" -) - -const ( - RLIMIT_CPU = iota // CPU time in sec - RLIMIT_FSIZE // Maximum filesize - RLIMIT_DATA // max data size - RLIMIT_STACK // max stack size - RLIMIT_CORE // max core file size - RLIMIT_RSS // max resident set size - RLIMIT_NPROC // max number of processes - RLIMIT_NOFILE // max number of open files - RLIMIT_MEMLOCK // max locked-in-memory address space - RLIMIT_AS // address space limit - RLIMIT_LOCKS // maximum file locks held - RLIMIT_SIGPENDING // max number of pending signals - RLIMIT_MSGQUEUE // maximum bytes in POSIX mqueues - RLIMIT_NICE // max nice prio allowed to raise to - RLIMIT_RTPRIO // maximum realtime priority - RLIMIT_RTTIME // timeout for RT tasks in us -) - -var rlimitMap = map[string]int{ - "RLIMIT_CPU": RLIMIT_CPU, - "RLIMIT_FSIZE": RLIMIT_FSIZE, - "RLIMIT_DATA": RLIMIT_DATA, - "RLIMIT_STACK": RLIMIT_STACK, - "RLIMIT_CORE": RLIMIT_CORE, - "RLIMIT_RSS": RLIMIT_RSS, - "RLIMIT_NPROC": RLIMIT_NPROC, - "RLIMIT_NOFILE": RLIMIT_NOFILE, - "RLIMIT_MEMLOCK": RLIMIT_MEMLOCK, - "RLIMIT_AS": RLIMIT_AS, - "RLIMIT_LOCKS": RLIMIT_LOCKS, - "RLIMIT_SGPENDING": RLIMIT_SIGPENDING, - "RLIMIT_MSGQUEUE": RLIMIT_MSGQUEUE, - "RLIMIT_NICE": RLIMIT_NICE, - "RLIMIT_RTPRIO": RLIMIT_RTPRIO, - "RLIMIT_RTTIME": RLIMIT_RTTIME, -} - -func strToRlimit(key string) (int, error) { - rl, ok := rlimitMap[key] - if !ok { - return 0, fmt.Errorf("Wrong rlimit value: %s", key) - } - return rl, nil -} - -const wildcard = -1 - -var allowedDevices = []*configs.Device{ - // allow mknod for any device - { - Type: 'c', - Major: wildcard, - Minor: wildcard, - Permissions: "m", - }, - { - Type: 'b', - Major: wildcard, - Minor: wildcard, - Permissions: "m", - }, - { - Path: "/dev/console", - Type: 'c', - Major: 5, - Minor: 1, - Permissions: "rwm", - }, - { - Path: "/dev/tty0", - Type: 'c', - Major: 4, - Minor: 0, - Permissions: "rwm", - }, - { - Path: "/dev/tty1", - Type: 'c', - Major: 4, - Minor: 1, - Permissions: "rwm", - }, - // /dev/pts/ - pts namespaces are "coming soon" - { - Path: "", - Type: 'c', - Major: 136, - Minor: wildcard, - Permissions: "rwm", - }, - { - Path: "", - Type: 'c', - Major: 5, - Minor: 2, - Permissions: "rwm", - }, - // tuntap - { - Path: "", - Type: 'c', - Major: 10, - Minor: 200, - Permissions: "rwm", - }, -} - -var namespaceMapping = map[specs.NamespaceType]configs.NamespaceType{ - specs.PIDNamespace: configs.NEWPID, - specs.NetworkNamespace: configs.NEWNET, - specs.MountNamespace: configs.NEWNS, - specs.UserNamespace: configs.NEWUSER, - specs.IPCNamespace: configs.NEWIPC, - specs.UTSNamespace: configs.NEWUTS, -} - -var mountPropagationMapping = map[string]int{ - "rprivate": syscall.MS_PRIVATE | syscall.MS_REC, - "private": syscall.MS_PRIVATE, - "rslave": syscall.MS_SLAVE | syscall.MS_REC, - "slave": syscall.MS_SLAVE, - "rshared": syscall.MS_SHARED | syscall.MS_REC, - "shared": syscall.MS_SHARED, - "": syscall.MS_PRIVATE | syscall.MS_REC, -} - -func init() { - if len(os.Args) > 1 && os.Args[1] == "init" { - goruntime.GOMAXPROCS(1) - goruntime.LockOSThread() - factory, _ := libcontainer.New("") - if err := factory.StartInitialization(); err != nil { - fmt.Fprint(os.Stderr, err) - os.Exit(1) - } - panic("--this line should have never been executed, congratulations--") - } -} - -type libcontainerProcess struct { - process *libcontainer.Process - spec specs.Process -} - -// change interface to support an error -func (p *libcontainerProcess) Pid() (int, error) { - pid, err := p.process.Pid() - if err != nil { - return -1, err - } - return pid, nil -} - -func (p *libcontainerProcess) Spec() specs.Process { - return p.spec -} - -func (p *libcontainerProcess) Signal(s os.Signal) error { - return p.process.Signal(s) -} - -func (p *libcontainerProcess) Close() error { - // in close we always need to call wait to close/flush any pipes - p.process.Wait() - // explicitly close any open fd on the process - for _, cl := range []interface{}{ - p.process.Stderr, - p.process.Stdout, - p.process.Stdin, - } { - if cl != nil { - if c, ok := cl.(io.Closer); ok { - c.Close() - } - } - } - return nil -} - -type libcontainerContainer struct { - c libcontainer.Container - initProcess *libcontainerProcess - additionalProcesses map[int]*libcontainerProcess - exitStatus int - exited bool - path string -} - -func (c *libcontainerContainer) Checkpoints() ([]runtime.Checkpoint, error) { - out := []runtime.Checkpoint{} - files, err := ioutil.ReadDir(c.getCheckpointPath("")) - if err != nil { - if os.IsNotExist(err) { - return out, nil - } - return nil, err - } - for _, fi := range files { - out = append(out, runtime.Checkpoint{ - Name: fi.Name(), - Timestamp: fi.ModTime(), - }) - } - return out, nil -} - -func (c *libcontainerContainer) DeleteCheckpoint(name string) error { - path := c.getCheckpointPath(name) - if err := os.RemoveAll(path); err != nil { - if os.IsNotExist(err) { - return runtime.ErrCheckpointNotExists - } - return err - } - return nil -} - -func (c *libcontainerContainer) getCheckpointPath(name string) string { - return filepath.Join(c.path, "checkpoints", name) -} - -func (c *libcontainerContainer) Checkpoint(cp runtime.Checkpoint) error { - opts := c.createCheckpointOpts(cp) - if err := os.MkdirAll(filepath.Dir(opts.ImagesDirectory), 0755); err != nil { - return err - } - // mkdir is atomic so if it already exists we can fail - if err := os.Mkdir(opts.ImagesDirectory, 0755); err != nil { - if os.IsExist(err) { - return runtime.ErrCheckpointExists - } - return err - } - if err := c.c.Checkpoint(opts); err != nil { - return err - } - return nil -} - -func (c *libcontainerContainer) createCheckpointOpts(cp runtime.Checkpoint) *libcontainer.CriuOpts { - opts := libcontainer.CriuOpts{} - opts.LeaveRunning = !cp.Exit - opts.ShellJob = cp.Shell - opts.TcpEstablished = cp.Tcp - opts.ExternalUnixConnections = cp.UnixSockets - opts.ImagesDirectory = c.getCheckpointPath(cp.Name) - return &opts -} - -func (c *libcontainerContainer) Restore(name string) error { - path := c.getCheckpointPath(name) - var opts libcontainer.CriuOpts - opts.ImagesDirectory = path - return c.c.Restore(c.initProcess.process, &opts) -} - -func (c *libcontainerContainer) Resume() error { - return c.c.Resume() -} - -func (c *libcontainerContainer) Pause() error { - return c.c.Pause() -} - -func (c *libcontainerContainer) State() runtime.State { - // TODO: what to do with error - state, err := c.c.Status() - if err != nil { - return runtime.State("") - } - switch state { - case libcontainer.Paused, libcontainer.Pausing: - return runtime.Paused - } - return runtime.State("") -} - -func (c *libcontainerContainer) ID() string { - return c.c.ID() -} - -func (c *libcontainerContainer) Path() string { - return c.path -} - -func (c *libcontainerContainer) Pid() (int, error) { - return c.initProcess.Pid() -} - -func (c *libcontainerContainer) Start() error { - return c.c.Start(c.initProcess.process) -} - -func (c *libcontainerContainer) SetExited(status int) { - c.exitStatus = status - // meh - c.exited = true - c.initProcess.Close() -} - -func (c *libcontainerContainer) Stats() (*runtime.Stat, error) { - now := time.Now() - stats, err := c.c.Stats() - if err != nil { - return nil, err - } - return &runtime.Stat{ - Timestamp: now, - Data: stats, - }, nil -} - -func (c *libcontainerContainer) Delete() error { - return c.c.Destroy() -} - -func (c *libcontainerContainer) Processes() ([]runtime.Process, error) { - procs := []runtime.Process{ - c.initProcess, - } - for _, p := range c.additionalProcesses { - procs = append(procs, p) - } - return procs, nil -} - -func (c *libcontainerContainer) RemoveProcess(pid int) error { - proc, ok := c.additionalProcesses[pid] - if !ok { - return runtime.ErrNotChildProcess - } - err := proc.Close() - delete(c.additionalProcesses, pid) - return err -} - -func (c *libcontainerContainer) OOM() (<-chan struct{}, error) { - return c.c.NotifyOOM() -} - -func NewRuntime(stateDir string) (runtime.Runtime, error) { - f, err := libcontainer.New(stateDir, libcontainer.Cgroupfs, func(l *libcontainer.LinuxFactory) error { - //l.CriuPath = context.GlobalString("criu") - return nil - }) - if err != nil { - return nil, err - } - return &libcontainerRuntime{ - factory: f, - }, nil -} - -type libcontainerRuntime struct { - factory libcontainer.Factory -} - -func (r *libcontainerRuntime) Type() string { - return "libcontainer" -} - -func (r *libcontainerRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) { - spec, rspec, err := r.loadSpec( - filepath.Join(bundlePath, "config.json"), - filepath.Join(bundlePath, "runtime.json"), - ) - if err != nil { - return nil, nil, err - } - config, err := r.createLibcontainerConfig(id, bundlePath, spec, rspec) - if err != nil { - return nil, nil, err - } - container, err := r.factory.Create(id, config) - if err != nil { - return nil, nil, fmt.Errorf("create container: %v", err) - } - process, err := r.newProcess(spec.Process) - if err != nil { - return nil, nil, err - } - var rio runtime.IO - if spec.Process.Terminal { - if err := process.ConsoleFromPath(consolePath); err != nil { - return nil, nil, err - } - } else { - uid, err := config.HostUID() - if err != nil { - return nil, nil, err - } - i, err := process.InitializeIO(uid) - if err != nil { - return nil, nil, err - } - rio.Stdin = i.Stdin - rio.Stderr = i.Stderr - rio.Stdout = i.Stdout - } - c := &libcontainerContainer{ - c: container, - additionalProcesses: make(map[int]*libcontainerProcess), - initProcess: &libcontainerProcess{ - process: process, - spec: spec.Process, - }, - path: bundlePath, - } - return c, &rio, nil -} - -func (r *libcontainerRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) { - c, ok := ci.(*libcontainerContainer) - if !ok { - return nil, nil, runtime.ErrInvalidContainerType - } - process, err := r.newProcess(p) - if err != nil { - return nil, nil, err - } - var rio runtime.IO - if p.Terminal { - if err := process.ConsoleFromPath(consolePath); err != nil { - return nil, nil, err - } - } else { - uid, err := c.c.Config().HostUID() - if err != nil { - return nil, nil, err - } - i, err := process.InitializeIO(uid) - if err != nil { - return nil, nil, err - } - rio.Stdin = i.Stdin - rio.Stderr = i.Stderr - rio.Stdout = i.Stdout - } - if err := c.c.Start(process); err != nil { - return nil, nil, err - } - lp := &libcontainerProcess{ - process: process, - spec: p, - } - pid, err := process.Pid() - if err != nil { - return nil, nil, err - } - c.additionalProcesses[pid] = lp - return lp, &rio, nil -} - -// newProcess returns a new libcontainer Process with the arguments from the -// spec and stdio from the current process. -func (r *libcontainerRuntime) newProcess(p specs.Process) (*libcontainer.Process, error) { - return &libcontainer.Process{ - Args: p.Args, - Env: p.Env, - // TODO: fix libcontainer's API to better support uid/gid in a typesafe way. - User: fmt.Sprintf("%d:%d", p.User.UID, p.User.GID), - Cwd: p.Cwd, - }, nil -} - -// loadSpec loads the specification from the provided path. -// If the path is empty then the default path will be "config.json" -func (r *libcontainerRuntime) loadSpec(cPath, rPath string) (spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec, err error) { - cf, err := os.Open(cPath) - if err != nil { - if os.IsNotExist(err) { - return nil, nil, fmt.Errorf("JSON specification file at %s not found", cPath) - } - return spec, rspec, err - } - defer cf.Close() - - rf, err := os.Open(rPath) - if err != nil { - if os.IsNotExist(err) { - return nil, nil, fmt.Errorf("JSON runtime config file at %s not found", rPath) - } - return spec, rspec, err - } - defer rf.Close() - - if err = json.NewDecoder(cf).Decode(&spec); err != nil { - return spec, rspec, fmt.Errorf("unmarshal %s: %v", cPath, err) - } - if err = json.NewDecoder(rf).Decode(&rspec); err != nil { - return spec, rspec, fmt.Errorf("unmarshal %s: %v", rPath, err) - } - return spec, rspec, r.checkSpecVersion(spec) -} - -// checkSpecVersion makes sure that the spec version matches runc's while we are in the initial -// development period. It is better to hard fail than have missing fields or options in the spec. -func (r *libcontainerRuntime) checkSpecVersion(s *specs.LinuxSpec) error { - if s.Version != specs.Version { - return fmt.Errorf("spec version is not compatible with implemented version %q: spec %q", specs.Version, s.Version) - } - return nil -} - -func (r *libcontainerRuntime) createLibcontainerConfig(cgroupName, bundlePath string, spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec) (*configs.Config, error) { - rootfsPath := spec.Root.Path - if !filepath.IsAbs(rootfsPath) { - rootfsPath = filepath.Join(bundlePath, rootfsPath) - } - config := &configs.Config{ - Rootfs: rootfsPath, - Capabilities: spec.Linux.Capabilities, - Readonlyfs: spec.Root.Readonly, - Hostname: spec.Hostname, - } - for _, ns := range rspec.Linux.Namespaces { - t, exists := namespaceMapping[ns.Type] - if !exists { - return nil, fmt.Errorf("namespace %q does not exist", ns) - } - config.Namespaces.Add(t, ns.Path) - } - if config.Namespaces.Contains(configs.NEWNET) { - config.Networks = []*configs.Network{ - { - Type: "loopback", - }, - } - } - for _, mp := range spec.Mounts { - m, ok := rspec.Mounts[mp.Name] - if !ok { - return nil, fmt.Errorf("Mount with Name %q not found in runtime config", mp.Name) - } - config.Mounts = append(config.Mounts, r.createLibcontainerMount(bundlePath, mp.Path, m)) - } - - // Convert rootfs propagation flag - if rspec.Linux.RootfsPropagation != "" { - _, pflags, _ := parseMountOptions([]string{rspec.Linux.RootfsPropagation}) - if len(pflags) == 1 { - config.RootPropagation = pflags[0] - } - } - - if err := r.createDevices(rspec, config); err != nil { - return nil, err - } - if err := r.setupUserNamespace(rspec, config); err != nil { - return nil, err - } - for _, rlimit := range rspec.Linux.Rlimits { - rl, err := r.createLibContainerRlimit(rlimit) - if err != nil { - return nil, err - } - config.Rlimits = append(config.Rlimits, rl) - } - c, err := r.createCgroupConfig(cgroupName, rspec, config.Devices) - if err != nil { - return nil, err - } - config.Cgroups = c - if config.Readonlyfs { - r.setReadonly(config) - config.MaskPaths = []string{ - "/proc/kcore", - } - config.ReadonlyPaths = []string{ - "/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus", - } - } - seccomp, err := r.setupSeccomp(&rspec.Linux.Seccomp) - if err != nil { - return nil, err - } - config.Seccomp = seccomp - config.Sysctl = rspec.Linux.Sysctl - config.ProcessLabel = rspec.Linux.SelinuxProcessLabel - config.AppArmorProfile = rspec.Linux.ApparmorProfile - for _, g := range spec.Process.User.AdditionalGids { - config.AdditionalGroups = append(config.AdditionalGroups, strconv.FormatUint(uint64(g), 10)) - } - r.createHooks(rspec, config) - config.Version = specs.Version - return config, nil -} - -func (r *libcontainerRuntime) createLibcontainerMount(cwd, dest string, m specs.Mount) *configs.Mount { - flags, pgflags, data := parseMountOptions(m.Options) - source := m.Source - if m.Type == "bind" { - if !filepath.IsAbs(source) { - source = filepath.Join(cwd, m.Source) - } - } - return &configs.Mount{ - Device: m.Type, - Source: source, - Destination: dest, - Data: data, - Flags: flags, - PropagationFlags: pgflags, - } -} - -func (rt *libcontainerRuntime) createCgroupConfig(name string, spec *specs.LinuxRuntimeSpec, devices []*configs.Device) (*configs.Cgroup, error) { - cr := &configs.Cgroup{ - Name: name, - Parent: "/containerd", - } - c := &configs.Resources{ - AllowedDevices: append(devices, allowedDevices...), - } - cr.Resources = c - r := spec.Linux.Resources - if r.Memory != nil { - if r.Memory.Limit != nil { - c.Memory = int64(*r.Memory.Limit) - } - if r.Memory.Reservation != nil { - c.MemoryReservation = int64(*r.Memory.Reservation) - } - if r.Memory.Swap != nil { - c.MemorySwap = int64(*r.Memory.Swap) - } - if r.Memory.Kernel != nil { - c.KernelMemory = int64(*r.Memory.Kernel) - } - if r.Memory.Swappiness != nil { - c.MemorySwappiness = int64(*r.Memory.Swappiness) - } - } - if r.CPU != nil { - if r.CPU.Shares != nil { - c.CpuShares = int64(*r.CPU.Shares) - } - if r.CPU.Quota != nil { - c.CpuQuota = int64(*r.CPU.Quota) - } - if r.CPU.Period != nil { - c.CpuPeriod = int64(*r.CPU.Period) - } - if r.CPU.RealtimeRuntime != nil { - c.CpuRtRuntime = int64(*r.CPU.RealtimeRuntime) - } - if r.CPU.RealtimePeriod != nil { - c.CpuRtPeriod = int64(*r.CPU.RealtimePeriod) - } - if r.CPU.Cpus != nil { - c.CpusetCpus = *r.CPU.Cpus - } - if r.CPU.Mems != nil { - c.CpusetMems = *r.CPU.Mems - } - } - if r.BlockIO != nil { - if r.BlockIO.Weight != nil { - c.BlkioWeight = *r.BlockIO.Weight - } - if r.BlockIO.LeafWeight != nil { - c.BlkioLeafWeight = *r.BlockIO.LeafWeight - } - } - for _, wd := range r.BlockIO.WeightDevice { - weightDevice := configs.NewWeightDevice(wd.Major, wd.Minor, *wd.Weight, *wd.LeafWeight) - c.BlkioWeightDevice = append(c.BlkioWeightDevice, weightDevice) - } - for _, td := range r.BlockIO.ThrottleReadBpsDevice { - throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate) - c.BlkioThrottleReadBpsDevice = append(c.BlkioThrottleReadBpsDevice, throttleDevice) - } - for _, td := range r.BlockIO.ThrottleWriteBpsDevice { - throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate) - c.BlkioThrottleWriteBpsDevice = append(c.BlkioThrottleWriteBpsDevice, throttleDevice) - } - for _, td := range r.BlockIO.ThrottleReadIOPSDevice { - throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate) - c.BlkioThrottleReadIOPSDevice = append(c.BlkioThrottleReadIOPSDevice, throttleDevice) - } - for _, td := range r.BlockIO.ThrottleWriteIOPSDevice { - throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate) - c.BlkioThrottleWriteIOPSDevice = append(c.BlkioThrottleWriteIOPSDevice, throttleDevice) - } - for _, l := range r.HugepageLimits { - c.HugetlbLimit = append(c.HugetlbLimit, &configs.HugepageLimit{ - Pagesize: *l.Pagesize, - Limit: *l.Limit, - }) - } - c.OomKillDisable = r.DisableOOMKiller != nil && *r.DisableOOMKiller - if r.Network != nil { - c.NetClsClassid = r.Network.ClassID - for _, m := range r.Network.Priorities { - c.NetPrioIfpriomap = append(c.NetPrioIfpriomap, &configs.IfPrioMap{ - Interface: m.Name, - Priority: int64(m.Priority), - }) - } - } - return cr, nil -} - -func (r *libcontainerRuntime) createDevices(spec *specs.LinuxRuntimeSpec, config *configs.Config) error { - for _, d := range spec.Linux.Devices { - device := &configs.Device{ - Type: d.Type, - Path: d.Path, - Major: d.Major, - Minor: d.Minor, - Permissions: d.Permissions, - FileMode: d.FileMode, - Uid: d.UID, - Gid: d.GID, - } - config.Devices = append(config.Devices, device) - } - return nil -} - -func (r *libcontainerRuntime) setReadonly(config *configs.Config) { - for _, m := range config.Mounts { - if m.Device == "sysfs" { - m.Flags |= syscall.MS_RDONLY - } - } -} - -func (r *libcontainerRuntime) setupUserNamespace(spec *specs.LinuxRuntimeSpec, config *configs.Config) error { - if len(spec.Linux.UIDMappings) == 0 { - return nil - } - config.Namespaces.Add(configs.NEWUSER, "") - create := func(m specs.IDMapping) configs.IDMap { - return configs.IDMap{ - HostID: int(m.HostID), - ContainerID: int(m.ContainerID), - Size: int(m.Size), - } - } - for _, m := range spec.Linux.UIDMappings { - config.UidMappings = append(config.UidMappings, create(m)) - } - for _, m := range spec.Linux.GIDMappings { - config.GidMappings = append(config.GidMappings, create(m)) - } - rootUID, err := config.HostUID() - if err != nil { - return err - } - rootGID, err := config.HostGID() - if err != nil { - return err - } - for _, node := range config.Devices { - node.Uid = uint32(rootUID) - node.Gid = uint32(rootGID) - } - return nil -} - -func (r *libcontainerRuntime) createLibContainerRlimit(rlimit specs.Rlimit) (configs.Rlimit, error) { - rl, err := strToRlimit(rlimit.Type) - if err != nil { - return configs.Rlimit{}, err - } - return configs.Rlimit{ - Type: rl, - Hard: uint64(rlimit.Hard), - Soft: uint64(rlimit.Soft), - }, nil -} - -// parseMountOptions parses the string and returns the flags, propagation -// flags and any mount data that it contains. -func parseMountOptions(options []string) (int, []int, string) { - var ( - flag int - pgflag []int - data []string - ) - flags := map[string]struct { - clear bool - flag int - }{ - "async": {true, syscall.MS_SYNCHRONOUS}, - "atime": {true, syscall.MS_NOATIME}, - "bind": {false, syscall.MS_BIND}, - "defaults": {false, 0}, - "dev": {true, syscall.MS_NODEV}, - "diratime": {true, syscall.MS_NODIRATIME}, - "dirsync": {false, syscall.MS_DIRSYNC}, - "exec": {true, syscall.MS_NOEXEC}, - "mand": {false, syscall.MS_MANDLOCK}, - "noatime": {false, syscall.MS_NOATIME}, - "nodev": {false, syscall.MS_NODEV}, - "nodiratime": {false, syscall.MS_NODIRATIME}, - "noexec": {false, syscall.MS_NOEXEC}, - "nomand": {true, syscall.MS_MANDLOCK}, - "norelatime": {true, syscall.MS_RELATIME}, - "nostrictatime": {true, syscall.MS_STRICTATIME}, - "nosuid": {false, syscall.MS_NOSUID}, - "rbind": {false, syscall.MS_BIND | syscall.MS_REC}, - "relatime": {false, syscall.MS_RELATIME}, - "remount": {false, syscall.MS_REMOUNT}, - "ro": {false, syscall.MS_RDONLY}, - "rw": {true, syscall.MS_RDONLY}, - "strictatime": {false, syscall.MS_STRICTATIME}, - "suid": {true, syscall.MS_NOSUID}, - "sync": {false, syscall.MS_SYNCHRONOUS}, - } - propagationFlags := map[string]struct { - clear bool - flag int - }{ - "private": {false, syscall.MS_PRIVATE}, - "shared": {false, syscall.MS_SHARED}, - "slave": {false, syscall.MS_SLAVE}, - "unbindable": {false, syscall.MS_UNBINDABLE}, - "rprivate": {false, syscall.MS_PRIVATE | syscall.MS_REC}, - "rshared": {false, syscall.MS_SHARED | syscall.MS_REC}, - "rslave": {false, syscall.MS_SLAVE | syscall.MS_REC}, - "runbindable": {false, syscall.MS_UNBINDABLE | syscall.MS_REC}, - } - for _, o := range options { - // If the option does not exist in the flags table or the flag - // is not supported on the platform, - // then it is a data value for a specific fs type - if f, exists := flags[o]; exists && f.flag != 0 { - if f.clear { - flag &= ^f.flag - } else { - flag |= f.flag - } - } else if f, exists := propagationFlags[o]; exists && f.flag != 0 { - pgflag = append(pgflag, f.flag) - } else { - data = append(data, o) - } - } - return flag, pgflag, strings.Join(data, ",") -} - -func (r *libcontainerRuntime) setupSeccomp(config *specs.Seccomp) (*configs.Seccomp, error) { - if config == nil { - return nil, nil - } - - // No default action specified, no syscalls listed, assume seccomp disabled - if config.DefaultAction == "" && len(config.Syscalls) == 0 { - return nil, nil - } - - newConfig := new(configs.Seccomp) - newConfig.Syscalls = []*configs.Syscall{} - - if len(config.Architectures) > 0 { - newConfig.Architectures = []string{} - for _, arch := range config.Architectures { - newArch, err := seccomp.ConvertStringToArch(string(arch)) - if err != nil { - return nil, err - } - newConfig.Architectures = append(newConfig.Architectures, newArch) - } - } - - // Convert default action from string representation - newDefaultAction, err := seccomp.ConvertStringToAction(string(config.DefaultAction)) - if err != nil { - return nil, err - } - newConfig.DefaultAction = newDefaultAction - - // Loop through all syscall blocks and convert them to libcontainer format - for _, call := range config.Syscalls { - newAction, err := seccomp.ConvertStringToAction(string(call.Action)) - if err != nil { - return nil, err - } - - newCall := configs.Syscall{ - Name: call.Name, - Action: newAction, - Args: []*configs.Arg{}, - } - - // Loop through all the arguments of the syscall and convert them - for _, arg := range call.Args { - newOp, err := seccomp.ConvertStringToOperator(string(arg.Op)) - if err != nil { - return nil, err - } - - newArg := configs.Arg{ - Index: arg.Index, - Value: arg.Value, - ValueTwo: arg.ValueTwo, - Op: newOp, - } - - newCall.Args = append(newCall.Args, &newArg) - } - - newConfig.Syscalls = append(newConfig.Syscalls, &newCall) - } - - return newConfig, nil -} - -func (r *libcontainerRuntime) createHooks(rspec *specs.LinuxRuntimeSpec, config *configs.Config) { - config.Hooks = &configs.Hooks{} - for _, h := range rspec.Hooks.Prestart { - cmd := configs.Command{ - Path: h.Path, - Args: h.Args, - Env: h.Env, - } - config.Hooks.Prestart = append(config.Hooks.Prestart, configs.NewCommandHook(cmd)) - } - for _, h := range rspec.Hooks.Poststop { - cmd := configs.Command{ - Path: h.Path, - Args: h.Args, - Env: h.Env, - } - config.Hooks.Poststop = append(config.Hooks.Poststop, configs.NewCommandHook(cmd)) - } -} diff --git a/runc/runc.go b/runc/runc.go deleted file mode 100644 index 75f48b17d..000000000 --- a/runc/runc.go +++ /dev/null @@ -1,235 +0,0 @@ -// +build runc - -package runc - -import ( - "encoding/json" - "errors" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - - "github.com/docker/containerd/runtime" - "github.com/opencontainers/specs" -) - -func NewRuntime(stateDir string) (runtime.Runtime, error) { - return &runcRuntime{ - stateDir: stateDir, - }, nil -} - -type runcContainer struct { - id string - path string - stateDir string - exitStatus int - processes map[int]*runcProcess - initProcess *runcProcess -} - -func (c *runcContainer) ID() string { - return c.id -} - -func (c *runcContainer) Start() error { - return c.initProcess.cmd.Start() -} - -func (c *runcContainer) Stats() (*runtime.Stat, error) { - return nil, errors.New("containerd: runc does not support stats in containerd") -} - -func (c *runcContainer) Path() string { - return c.path -} - -func (c *runcContainer) Pid() (int, error) { - return c.initProcess.cmd.Process.Pid, nil -} - -func (c *runcContainer) SetExited(status int) { - c.exitStatus = status -} - -// noop for runc -func (c *runcContainer) Delete() error { - return nil -} - -func (c *runcContainer) Processes() ([]runtime.Process, error) { - procs := []runtime.Process{ - c.initProcess, - } - for _, p := range c.processes { - procs = append(procs, p) - } - return procs, nil -} - -func (c *runcContainer) RemoveProcess(pid int) error { - if _, ok := c.processes[pid]; !ok { - return runtime.ErrNotChildProcess - } - delete(c.processes, pid) - return nil -} - -func (c *runcContainer) State() runtime.State { - // TODO: how to do this with runc - return runtime.State{ - Status: runtime.Running, - } -} - -func (c *runcContainer) Resume() error { - return c.newCommand("resume").Run() -} - -func (c *runcContainer) Pause() error { - return c.newCommand("pause").Run() -} - -// TODO: pass arguments -func (c *runcContainer) Checkpoint(runtime.Checkpoint) error { - return c.newCommand("checkpoint").Run() -} - -// TODO: pass arguments -func (c *runcContainer) Restore(cp string) error { - return c.newCommand("restore").Run() -} - -// TODO: pass arguments -func (c *runcContainer) DeleteCheckpoint(cp string) error { - return errors.New("not implemented") -} - -// TODO: implement in runc -func (c *runcContainer) Checkpoints() ([]runtime.Checkpoint, error) { - return nil, errors.New("not implemented") -} - -func (c *runcContainer) OOM() (<-chan struct{}, error) { - return nil, errors.New("not implemented") -} - -func (c *runcContainer) newCommand(args ...string) *exec.Cmd { - cmd := exec.Command("runc", append([]string{"--root", c.stateDir, "--id", c.id}, args...)...) - cmd.Dir = c.path - return cmd -} - -type runcProcess struct { - cmd *exec.Cmd - spec specs.Process -} - -// pid of the container, not of runc -func (p *runcProcess) Pid() (int, error) { - return p.cmd.Process.Pid, nil -} - -func (p *runcProcess) Spec() specs.Process { - return p.spec -} - -func (p *runcProcess) Signal(s os.Signal) error { - return p.cmd.Process.Signal(s) -} - -func (p *runcProcess) Close() error { - return nil -} - -type runcRuntime struct { - stateDir string -} - -func (r *runcRuntime) Type() string { - return "runc" -} - -func (r *runcRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) { - var s specs.Spec - f, err := os.Open(filepath.Join(bundlePath, "config.json")) - if err != nil { - return nil, nil, err - } - defer f.Close() - - if err := json.NewDecoder(f).Decode(&s); err != nil { - return nil, nil, err - } - cmd := exec.Command("runc", "--root", r.stateDir, "--id", id, "start") - cmd.Dir = bundlePath - i, err := r.createIO(cmd) - if err != nil { - return nil, nil, err - } - return &runcContainer{ - id: id, - path: bundlePath, - stateDir: r.stateDir, - initProcess: &runcProcess{ - cmd: cmd, - spec: s.Process, - }, - processes: make(map[int]*runcProcess), - }, i, nil -} - -func (r *runcRuntime) createIO(cmd *exec.Cmd) (*runtime.IO, error) { - w, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - ro, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - re, err := cmd.StderrPipe() - if err != nil { - return nil, err - } - return &runtime.IO{ - Stdin: w, - Stdout: ro, - Stderr: re, - }, nil -} - -func (r *runcRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) { - c, ok := ci.(*runcContainer) - if !ok { - return nil, nil, runtime.ErrInvalidContainerType - } - f, err := ioutil.TempFile("", "containerd") - if err != nil { - return nil, nil, err - } - err = json.NewEncoder(f).Encode(p) - f.Close() - if err != nil { - return nil, nil, err - } - cmd := c.newCommand("exec", f.Name()) - i, err := r.createIO(cmd) - if err != nil { - return nil, nil, err - } - process := &runcProcess{ - cmd: cmd, - spec: p, - } - if err := cmd.Start(); err != nil { - return nil, nil, err - } - pid, err := process.Pid() - if err != nil { - return nil, nil, err - } - c.processes[pid] = process - return process, i, nil -} diff --git a/runtime/container.go b/runtime/container.go index a725dbdc0..7c7f1ef78 100644 --- a/runtime/container.go +++ b/runtime/container.go @@ -10,9 +10,27 @@ import ( type Process interface { io.Closer - Pid() (int, error) + + // ID of the process. + // This is either "init" when it is the container's init process or + // it is a user provided id for the process similar to the container id + ID() string + // Stdin returns the path the the processes stdin fifo + Stdin() string + // Stdout returns the path the the processes stdout fifo + Stdout() string + // Stderr returns the path the the processes stderr fifo + Stderr() string + // ExitFD returns the fd the provides an event when the process exits + ExitFD() int + // ExitStatus returns the exit status of the process or an error if it + // has not exited + ExitStatus() (int, error) Spec() specs.Process + // Signal sends the provided signal to the process Signal(os.Signal) error + // Container returns the container that the process belongs to + Container() Container } type State string @@ -77,20 +95,16 @@ type Checkpoint struct { type Container interface { // ID returns the container ID ID() string - // Start starts the init process of the container - Start() error // Path returns the path to the bundle Path() string - // Pid returns the container's init process id - Pid() (int, error) - // SetExited sets the exit status of the container after its init dies - SetExited(status int) - // Delete deletes the container + // Start starts the init process of the container + Start() (Process, error) + // Delete removes the container's state and any resources Delete() error + // Pid returns the container's init process id + // Pid() (int, error) // Processes returns all the containers processes that have been added Processes() ([]Process, error) - // RemoveProcess removes a specific process for the container because it exited - RemoveProcess(pid int) error // State returns the containers runtime state State() State // Resume resumes a paused container @@ -98,15 +112,15 @@ type Container interface { // Pause pauses a running container Pause() error // Checkpoints returns all the checkpoints for a container - Checkpoints() ([]Checkpoint, error) + // Checkpoints() ([]Checkpoint, error) // Checkpoint creates a new checkpoint - Checkpoint(Checkpoint) error + // Checkpoint(Checkpoint) error // DeleteCheckpoint deletes the checkpoint for the provided name - DeleteCheckpoint(name string) error + // DeleteCheckpoint(name string) error // Restore restores the container to that of the checkpoint provided by name - Restore(name string) error + // Restore(name string) error // Stats returns realtime container stats and resource information - Stats() (*Stat, error) + // Stats() (*Stat, error) // OOM signals the channel if the container received an OOM notification - OOM() (<-chan struct{}, error) + // OOM() (<-chan struct{}, error) } diff --git a/runtime/lib.go b/runtime/lib.go new file mode 100644 index 000000000..f8b926c98 --- /dev/null +++ b/runtime/lib.go @@ -0,0 +1,172 @@ +package runtime + +import ( + "encoding/json" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/opencontainers/specs" +) + +const ( + ExitFile = "exit" + ExitStatusFile = "exitStatus" + StateFile = "state.json" + InitProcessID = "init" +) + +type state struct { + Bundle string `json:"bundle"` +} + +// New returns a new container +func New(root, id, bundle string) (Container, error) { + c := &container{ + root: root, + id: id, + bundle: bundle, + processes: make(map[string]*process), + } + if err := os.Mkdir(filepath.Join(root, id), 0755); err != nil { + return nil, err + } + f, err := os.Create(filepath.Join(root, id, StateFile)) + if err != nil { + return nil, err + } + defer f.Close() + if err := json.NewEncoder(f).Encode(state{ + Bundle: bundle, + }); err != nil { + return nil, err + } + return c, nil +} + +func Load(root, id string) (Container, error) { + var s state + f, err := os.Open(filepath.Join(root, id, StateFile)) + if err != nil { + return nil, err + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&s); err != nil { + return nil, err + } + c := &container{ + root: root, + id: id, + bundle: s.Bundle, + processes: make(map[string]*process), + } + dirs, err := ioutil.ReadDir(filepath.Join(root, id)) + if err != nil { + return nil, err + } + for _, d := range dirs { + if !d.IsDir() { + continue + } + pid := d.Name() + // TODO: get the process spec from a state file in the process dir + p, err := loadProcess(filepath.Join(root, id, pid), pid, c, specs.Process{}) + if err != nil { + if err == ErrProcessExited { + logrus.WithField("id", id).WithField("pid", pid).Debug("containerd: process exited while away") + // TODO: fire events to do the removal + if err := os.RemoveAll(filepath.Join(root, id, pid)); err != nil { + logrus.WithField("error", err).Warn("containerd: remove process state") + } + continue + } + return nil, err + } + c.processes[pid] = p + } + if len(c.processes) == 0 { + return nil, ErrContainerExited + } + return c, nil +} + +type container struct { + // path to store runtime state information + root string + id string + bundle string + processes map[string]*process +} + +func (c *container) ID() string { + return c.id +} + +func (c *container) Path() string { + return c.bundle +} + +func (c *container) Start() (Process, error) { + processRoot := filepath.Join(c.root, c.id, InitProcessID) + if err := os.MkdirAll(processRoot, 0755); err != nil { + return nil, err + } + cmd := exec.Command("containerd-shim", processRoot, c.id) + cmd.Dir = c.bundle + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + spec, err := c.readSpec() + if err != nil { + return nil, err + } + p, err := newProcess(processRoot, InitProcessID, c, spec.Process) + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + c.processes[InitProcessID] = p + return p, nil +} + +func (c *container) readSpec() (*specs.LinuxSpec, error) { + var spec specs.LinuxSpec + f, err := os.Open(filepath.Join(c.bundle, "config.json")) + if err != nil { + return nil, err + } + defer f.Close() + if err := json.NewDecoder(f).Decode(&spec); err != nil { + return nil, err + } + return &spec, nil +} + +func (c *container) Pause() error { + return errNotImplemented +} + +func (c *container) Resume() error { + return errNotImplemented +} + +func (c *container) State() State { + return Running +} + +func (c *container) Delete() error { + return os.RemoveAll(filepath.Join(c.root, c.id)) +} + +func (c *container) Processes() ([]Process, error) { + out := []Process{} + for _, p := range c.processes { + out = append(out, p) + } + return out, nil +} diff --git a/runtime/process.go b/runtime/process.go new file mode 100644 index 000000000..f094231fe --- /dev/null +++ b/runtime/process.go @@ -0,0 +1,141 @@ +package runtime + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "syscall" + + "github.com/opencontainers/specs" +) + +func newProcess(root, id string, c *container, s specs.Process) (*process, error) { + p := &process{ + root: root, + id: id, + container: c, + spec: s, + } + // create fifo's for the process + for name, fd := range map[string]*string{ + "stdin": &p.stdin, + "stdout": &p.stdout, + "stderr": &p.stderr, + } { + path := filepath.Join(root, name) + if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) { + return nil, err + } + *fd = path + } + exit, err := getExitPipe(filepath.Join(root, ExitFile)) + if err != nil { + return nil, err + } + p.exitPipe = exit + return p, nil +} + +func loadProcess(root, id string, c *container, s specs.Process) (*process, error) { + p := &process{ + root: root, + id: id, + container: c, + spec: s, + stdin: filepath.Join(root, "stdin"), + stdout: filepath.Join(root, "stdout"), + stderr: filepath.Join(root, "stderr"), + } + if _, err := p.ExitStatus(); err != nil { + if err == ErrProcessNotExited { + exit, err := getExitPipe(filepath.Join(root, ExitFile)) + if err != nil { + return nil, err + } + p.exitPipe = exit + return p, nil + } + return nil, err + } + return nil, ErrProcessExited +} + +func getExitPipe(path string) (*os.File, error) { + if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) { + return nil, err + } + // add NONBLOCK in case the other side has already closed or else + // this function would never return + return os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) +} + +type process struct { + root string + id string + // stdio fifos + stdin string + stdout string + stderr string + + exitPipe *os.File + container *container + spec specs.Process +} + +func (p *process) ID() string { + return p.id +} + +func (p *process) Container() Container { + return p.container +} + +// ExitFD returns the fd of the exit pipe +func (p *process) ExitFD() int { + return int(p.exitPipe.Fd()) +} + +func (p *process) ExitStatus() (int, error) { + data, err := ioutil.ReadFile(filepath.Join(p.root, ExitStatusFile)) + if err != nil { + if os.IsNotExist(err) { + return -1, ErrProcessNotExited + } + return -1, err + } + if len(data) == 0 { + return -1, ErrProcessNotExited + } + i, err := strconv.Atoi(string(data)) + if err != nil { + return -1, err + } + return i, nil +} + +// Signal sends the provided signal to the process +func (p *process) Signal(s os.Signal) error { + return errNotImplemented +} + +func (p *process) Spec() specs.Process { + return p.spec +} + +func (p *process) Stdin() string { + return p.stdin +} + +func (p *process) Stdout() string { + return p.stdout +} + +func (p *process) Stderr() string { + return p.stderr +} + +// Close closes any open files and/or resouces on the process +func (p *process) Close() error { + return p.exitPipe.Close() +} diff --git a/runtime/runtime.go b/runtime/runtime.go index 60abff62b..d4a35a8c7 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -1,10 +1,6 @@ package runtime -import ( - "errors" - - "github.com/opencontainers/specs" -) +import "errors" var ( ErrNotChildProcess = errors.New("containerd: not a child process for container") @@ -13,14 +9,8 @@ var ( ErrCheckpointExists = errors.New("containerd: checkpoint already exists") ErrContainerExited = errors.New("containerd: container has exited") ErrTerminalsNotSupported = errors.New("containerd: terminals are not supported for runtime") -) + ErrProcessNotExited = errors.New("containerd: process has not exited") + ErrProcessExited = errors.New("containerd: process has exited") -// Runtime handles containers, containers handle their own actions -type Runtime interface { - // Type of the runtime - Type() string - // Create creates a new container initialized but without it starting it - Create(id, bundlePath, consolePath string) (Container, *IO, error) - // StartProcess adds a new process to the container - StartProcess(c Container, p specs.Process, consolePath string) (Process, *IO, error) -} + errNotImplemented = errors.New("containerd: not implemented") +) diff --git a/supervisor/add_process.go b/supervisor/add_process.go index bc0efb9b1..3a0739131 100644 --- a/supervisor/add_process.go +++ b/supervisor/add_process.go @@ -1,11 +1,5 @@ package supervisor -import ( - "time" - - "github.com/Sirupsen/logrus" -) - type AddProcessEvent struct { s *Supervisor } @@ -13,30 +7,32 @@ type AddProcessEvent struct { // TODO: add this to worker for concurrent starts??? maybe not because of races where the container // could be stopped and removed... func (h *AddProcessEvent) Handle(e *Event) error { - start := time.Now() - ci, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - p, io, err := h.s.runtime.StartProcess(ci.container, *e.Process, e.Console) - if err != nil { - return err - } - if e.Pid, err = p.Pid(); err != nil { - return err - } - h.s.processes[e.Pid] = &containerInfo{ - container: ci.container, - } - l, err := h.s.copyIO(e.Stdin, e.Stdout, e.Stderr, io) - if err != nil { - // log the error but continue with the other commands - logrus.WithFields(logrus.Fields{ - "error": err, - "id": e.ID, - }).Error("log stdio") - } - h.s.processes[e.Pid].copier = l - ExecProcessTimer.UpdateSince(start) + /* + start := time.Now() + ci, ok := h.s.containers[e.ID] + if !ok { + return ErrContainerNotFound + } + p, io, err := h.s.runtime.StartProcess(ci.container, *e.Process, e.Console) + if err != nil { + return err + } + if e.Pid, err = p.Pid(); err != nil { + return err + } + h.s.processes[e.Pid] = &containerInfo{ + container: ci.container, + } + l, err := h.s.copyIO(e.Stdin, e.Stdout, e.Stderr, io) + if err != nil { + // log the error but continue with the other commands + logrus.WithFields(logrus.Fields{ + "error": err, + "id": e.ID, + }).Error("log stdio") + } + h.s.processes[e.Pid].copier = l + ExecProcessTimer.UpdateSince(start) + */ return nil } diff --git a/supervisor/checkpoint.go b/supervisor/checkpoint.go index a5ae54088..d65febf84 100644 --- a/supervisor/checkpoint.go +++ b/supervisor/checkpoint.go @@ -5,11 +5,14 @@ type CreateCheckpointEvent struct { } func (h *CreateCheckpointEvent) Handle(e *Event) error { - i, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - return i.container.Checkpoint(*e.Checkpoint) + /* + i, ok := h.s.containers[e.ID] + if !ok { + return ErrContainerNotFound + } + */ + return nil + // return i.container.Checkpoint(*e.Checkpoint) } type DeleteCheckpointEvent struct { @@ -17,9 +20,12 @@ type DeleteCheckpointEvent struct { } func (h *DeleteCheckpointEvent) Handle(e *Event) error { - i, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - return i.container.DeleteCheckpoint(e.Checkpoint.Name) + /* + i, ok := h.s.containers[e.ID] + if !ok { + return ErrContainerNotFound + } + */ + return nil + // return i.container.DeleteCheckpoint(e.Checkpoint.Name) } diff --git a/supervisor/create.go b/supervisor/create.go index 41f4c8b53..6e307b744 100644 --- a/supervisor/create.go +++ b/supervisor/create.go @@ -1,6 +1,10 @@ package supervisor -import "time" +import ( + "time" + + "github.com/docker/containerd/runtime" +) type StartEvent struct { s *Supervisor @@ -8,22 +12,17 @@ type StartEvent struct { func (h *StartEvent) Handle(e *Event) error { start := time.Now() - container, io, err := h.s.runtime.Create(e.ID, e.BundlePath, e.Console) + container, err := runtime.New(h.s.stateDir, e.ID, e.BundlePath) if err != nil { return err } - h.s.containerGroup.Add(1) h.s.containers[e.ID] = &containerInfo{ container: container, } ContainersCounter.Inc(1) task := &StartTask{ Err: e.Err, - IO: io, Container: container, - Stdin: e.Stdin, - Stdout: e.Stdout, - Stderr: e.Stderr, StartResponse: e.StartResponse, } if e.Checkpoint != nil { diff --git a/supervisor/delete.go b/supervisor/delete.go index 4651ef89f..1888ab164 100644 --- a/supervisor/delete.go +++ b/supervisor/delete.go @@ -29,7 +29,6 @@ func (h *DeleteEvent) Handle(e *Event) error { Pid: e.Pid, }) ContainersCounter.Dec(1) - h.s.containerGroup.Done() ContainerDeleteTimer.UpdateSince(start) } return nil diff --git a/supervisor/event.go b/supervisor/event.go index ac7ce00fe..63c205549 100644 --- a/supervisor/event.go +++ b/supervisor/event.go @@ -36,7 +36,9 @@ func NewEvent(t EventType) *Event { } type StartResponse struct { - Pid int + Stdin string + Stdout string + Stderr string } type Event struct { @@ -48,11 +50,12 @@ type Event struct { Stderr string Stdin string Console string - Pid int + Pid string Status int Signal os.Signal - Process *specs.Process + Process runtime.Process State runtime.State + ProcessSpec *specs.Process Containers []runtime.Container Checkpoint *runtime.Checkpoint Err chan error diff --git a/supervisor/exit.go b/supervisor/exit.go index 93a06c15c..fecfed84b 100644 --- a/supervisor/exit.go +++ b/supervisor/exit.go @@ -4,6 +4,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/containerd/runtime" ) type ExitEvent struct { @@ -12,36 +13,36 @@ type ExitEvent struct { func (h *ExitEvent) Handle(e *Event) error { start := time.Now() - logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}). - Debug("containerd: process exited") - // is it the child process of a container - if info, ok := h.s.processes[e.Pid]; ok { - ne := NewEvent(ExecExitEventType) - ne.ID = info.container.ID() - ne.Pid = e.Pid - ne.Status = e.Status - h.s.SendEvent(ne) - return nil - } - // is it the main container's process - container, err := h.s.getContainerForPid(e.Pid) + proc := e.Process + status, err := proc.ExitStatus() if err != nil { - if err != errNoContainerForPid { - logrus.WithField("error", err).Error("containerd: find containers main pid") - } + logrus.WithField("error", err).Error("containerd: get exit status") + } + logrus.WithFields(logrus.Fields{"pid": proc.ID(), "status": status}).Debug("containerd: process exited") + + // if the process is the the init process of the container then + // fire a separate event for this process + if proc.ID() != runtime.InitProcessID { + ne := NewEvent(ExecExitEventType) + ne.ID = proc.Container().ID() + ne.Status = status + h.s.SendEvent(ne) + return nil } - container.SetExited(e.Status) + container := proc.Container() ne := NewEvent(DeleteEventType) ne.ID = container.ID() - ne.Pid = e.Pid - ne.Status = e.Status + ne.Status = status + ne.Pid = proc.ID() h.s.SendEvent(ne) + // remove stats collection for container stopCollect := NewEvent(StopStatsEventType) stopCollect.ID = container.ID() h.s.SendEvent(stopCollect) ExitProcessTimer.UpdateSince(start) + return nil } @@ -51,14 +52,16 @@ type ExecExitEvent struct { func (h *ExecExitEvent) Handle(e *Event) error { // exec process: we remove this process without notifying the main event loop - info := h.s.processes[e.Pid] - if err := info.container.RemoveProcess(e.Pid); err != nil { - logrus.WithField("error", err).Error("containerd: find container for pid") - } - if err := info.copier.Close(); err != nil { - logrus.WithField("error", err).Error("containerd: close process IO") - } - delete(h.s.processes, e.Pid) - h.s.notifySubscribers(e) + /* + info := h.s.processes[e.Pid] + if err := info.container.RemoveProcess(e.Pid); err != nil { + logrus.WithField("error", err).Error("containerd: find container for pid") + } + if err := info.copier.Close(); err != nil { + logrus.WithField("error", err).Error("containerd: close process IO") + } + delete(h.s.processes, e.Pid) + h.s.notifySubscribers(e) + */ return nil } diff --git a/supervisor/machine.go b/supervisor/machine.go index e25932ab3..afc448f09 100644 --- a/supervisor/machine.go +++ b/supervisor/machine.go @@ -3,15 +3,12 @@ package supervisor import "github.com/cloudfoundry/gosigar" type Machine struct { - ID string Cpus int Memory int64 } -func CollectMachineInformation(id string) (Machine, error) { - m := Machine{ - ID: id, - } +func CollectMachineInformation() (Machine, error) { + m := Machine{} cpu := sigar.CpuList{} if err := cpu.Get(); err != nil { return m, err diff --git a/supervisor/monitor.go b/supervisor/monitor.go new file mode 100644 index 000000000..6cd191d42 --- /dev/null +++ b/supervisor/monitor.go @@ -0,0 +1,86 @@ +package supervisor + +import ( + "sync" + "syscall" + + "github.com/Sirupsen/logrus" + "github.com/docker/containerd/runtime" +) + +func NewMonitor() (*Monitor, error) { + m := &Monitor{ + processes: make(map[int]runtime.Process), + exits: make(chan runtime.Process, 1024), + } + fd, err := syscall.EpollCreate1(0) + if err != nil { + return nil, err + } + m.epollFd = fd + go m.start() + return m, nil +} + +type Monitor struct { + m sync.Mutex + processes map[int]runtime.Process + exits chan runtime.Process + epollFd int +} + +func (m *Monitor) Exits() chan runtime.Process { + return m.exits +} + +func (m *Monitor) Monitor(p runtime.Process) error { + m.m.Lock() + defer m.m.Unlock() + fd := p.ExitFD() + event := syscall.EpollEvent{ + Fd: int32(fd), + Events: syscall.EPOLLHUP, + } + if err := syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_ADD, fd, &event); err != nil { + return err + } + m.processes[fd] = p + return nil +} + +func (m *Monitor) Close() error { + return syscall.Close(m.epollFd) +} + +func (m *Monitor) start() { + var events [128]syscall.EpollEvent + for { + n, err := syscall.EpollWait(m.epollFd, events[:], -1) + if err != nil { + if err == syscall.EINTR { + continue + } + logrus.WithField("error", err).Fatal("containerd: epoll wait") + } + // process events + for i := 0; i < n; i++ { + if events[i].Events == syscall.EPOLLHUP { + fd := int(events[i].Fd) + m.m.Lock() + proc := m.processes[fd] + delete(m.processes, fd) + if err = syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_DEL, fd, &syscall.EpollEvent{ + Events: syscall.EPOLLHUP, + Fd: int32(fd), + }); err != nil { + logrus.WithField("error", err).Fatal("containerd: epoll remove fd") + } + if err := proc.Close(); err != nil { + logrus.WithField("error", err).Error("containerd: close process IO") + } + m.m.Unlock() + m.exits <- proc + } + } + } +} diff --git a/supervisor/signal.go b/supervisor/signal.go index 7d0fcaba9..51df0edb8 100644 --- a/supervisor/signal.go +++ b/supervisor/signal.go @@ -5,18 +5,20 @@ type SignalEvent struct { } func (h *SignalEvent) Handle(e *Event) error { - i, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - processes, err := i.container.Processes() - if err != nil { - return err - } - for _, p := range processes { - if pid, err := p.Pid(); err == nil && pid == e.Pid { - return p.Signal(e.Signal) + /* + i, ok := h.s.containers[e.ID] + if !ok { + return ErrContainerNotFound } - } + processes, err := i.container.Processes() + if err != nil { + return err + } + for _, p := range processes { + if pid, err := p.Pid(); err == nil && pid == e.Pid { + return p.Signal(e.Signal) + } + } + */ return ErrProcessNotFound } diff --git a/supervisor/stats_collector.go b/supervisor/stats_collector.go index f45b4ae17..178458b1e 100644 --- a/supervisor/stats_collector.go +++ b/supervisor/stats_collector.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/containerd/api/grpc/types" "github.com/docker/containerd/runtime" "github.com/docker/docker/pkg/pubsub" @@ -179,15 +178,17 @@ func (s *statsCollector) run() { continue } - for _, pair := range pairs { - stats, err := pair.ct.Stats() - if err != nil { - logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID()) - continue - } + /* + for _, pair := range pairs { + stats, err := pair.ct.Stats() + if err != nil { + logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID()) + continue + } - pair.pub.Publish(convertToPb(stats)) - } + pair.pub.Publish(convertToPb(stats)) + } + */ } } diff --git a/supervisor/supervisor.go b/supervisor/supervisor.go index 5c54a44d8..652d1c6ad 100644 --- a/supervisor/supervisor.go +++ b/supervisor/supervisor.go @@ -1,18 +1,16 @@ package supervisor import ( + "io/ioutil" "os" - "os/signal" "path/filepath" "sync" - "syscall" "time" "github.com/Sirupsen/logrus" "github.com/docker/containerd/chanotify" "github.com/docker/containerd/eventloop" "github.com/docker/containerd/runtime" - "github.com/opencontainers/runc/libcontainer" ) const ( @@ -21,29 +19,27 @@ const ( ) // New returns an initialized Process supervisor. -func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) { +func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) { if err := os.MkdirAll(stateDir, 0755); err != nil { return nil, err } - // register counters - r, err := newRuntime(filepath.Join(stateDir, id)) + machine, err := CollectMachineInformation() if err != nil { return nil, err } - machine, err := CollectMachineInformation(id) + monitor, err := NewMonitor() if err != nil { return nil, err } s := &Supervisor{ stateDir: stateDir, containers: make(map[string]*containerInfo), - processes: make(map[int]*containerInfo), - runtime: r, tasks: tasks, machine: machine, subscribers: make(map[chan *Event]struct{}), statsCollector: newStatsCollector(statsInterval), el: eventloop.NewChanLoop(defaultBufferSize), + monitor: monitor, } if oom { s.notifier = chanotify.New() @@ -71,7 +67,10 @@ func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, err UnsubscribeStatsEventType: &UnsubscribeStatsEvent{s}, StopStatsEventType: &StopStatsEvent{s}, } - // start the container workers for concurrent container starts + go s.exitHandler() + if err := s.restore(); err != nil { + return nil, err + } return s, nil } @@ -84,9 +83,7 @@ type Supervisor struct { // stateDir is the directory on the system to store container runtime state information. stateDir string containers map[string]*containerInfo - processes map[int]*containerInfo handlers map[EventType]Handler - runtime runtime.Runtime events chan *Event tasks chan *StartTask // we need a lock around the subscribers map only because additions and deletions from @@ -94,51 +91,18 @@ type Supervisor struct { subscriberLock sync.RWMutex subscribers map[chan *Event]struct{} machine Machine - containerGroup sync.WaitGroup statsCollector *statsCollector notifier *chanotify.Notifier el eventloop.EventLoop + monitor *Monitor } // Stop closes all tasks and sends a SIGTERM to each container's pid1 then waits for they to // terminate. After it has handled all the SIGCHILD events it will close the signals chan // and exit. Stop is a non-blocking call and will return after the containers have been signaled -func (s *Supervisor) Stop(sig chan os.Signal) { +func (s *Supervisor) Stop() { // Close the tasks channel so that no new containers get started close(s.tasks) - // send a SIGTERM to all containers - for id, i := range s.containers { - c := i.container - logrus.WithField("id", id).Debug("sending TERM to container processes") - procs, err := c.Processes() - if err != nil { - logrus.WithField("id", id).Warn("get container processes") - continue - } - if len(procs) == 0 { - continue - } - mainProc := procs[0] - if err := mainProc.Signal(syscall.SIGTERM); err != nil { - pid, _ := mainProc.Pid() - logrus.WithFields(logrus.Fields{ - "id": id, - "pid": pid, - "error": err, - }).Error("send SIGTERM to process") - } - } - go func() { - logrus.Debug("waiting for containers to exit") - s.containerGroup.Wait() - logrus.Debug("all containers exited") - if s.notifier != nil { - s.notifier.Close() - } - // stop receiving signals and close the channel - signal.Stop(sig) - close(sig) - }() } // Close closes any open files in the supervisor but expects that Stop has been @@ -190,7 +154,6 @@ func (s *Supervisor) notifySubscribers(e *Event) { // state of the Supervisor func (s *Supervisor) Start() error { logrus.WithFields(logrus.Fields{ - "runtime": s.runtime.Type(), "stateDir": s.stateDir, }).Debug("Supervisor started") return s.el.Start() @@ -202,45 +165,60 @@ func (s *Supervisor) Machine() Machine { return s.machine } -// getContainerForPid returns the container where the provided pid is the pid1 or main -// process in the container -func (s *Supervisor) getContainerForPid(pid int) (runtime.Container, error) { - for _, i := range s.containers { - container := i.container - cpid, err := container.Pid() - if err != nil { - if lerr, ok := err.(libcontainer.Error); ok { - if lerr.Code() == libcontainer.ProcessNotExecuted { - continue - } - } - logrus.WithField("error", err).Error("containerd: get container pid") - } - if pid == cpid { - return container, nil - } - } - return nil, errNoContainerForPid -} - // SendEvent sends the provided event the the supervisors main event loop func (s *Supervisor) SendEvent(evt *Event) { EventsCounter.Inc(1) s.el.Send(&commonEvent{data: evt, sv: s}) } -func (s *Supervisor) copyIO(stdin, stdout, stderr string, i *runtime.IO) (*copier, error) { - config := &ioConfig{ - Stdin: i.Stdin, - Stdout: i.Stdout, - Stderr: i.Stderr, - StdoutPath: stdout, - StderrPath: stderr, - StdinPath: stdin, +func (s *Supervisor) exitHandler() { + for p := range s.monitor.Exits() { + e := NewEvent(ExitEventType) + e.Process = p + s.SendEvent(e) } - l, err := newCopier(config) - if err != nil { - return nil, err - } - return l, nil +} + +func (s *Supervisor) monitorProcess(p runtime.Process) error { + return s.monitor.Monitor(p) +} + +func (s *Supervisor) restore() error { + dirs, err := ioutil.ReadDir(s.stateDir) + if err != nil { + return err + } + for _, d := range dirs { + if !d.IsDir() { + continue + } + id := d.Name() + container, err := runtime.Load(s.stateDir, id) + if err != nil { + if err == runtime.ErrContainerExited { + logrus.WithField("id", id).Debug("containerd: container exited while away") + // TODO: fire events to do the removal + if err := os.RemoveAll(filepath.Join(s.stateDir, id)); err != nil { + logrus.WithField("error", err).Warn("containerd: remove container state") + } + continue + } + return err + } + processes, err := container.Processes() + if err != nil { + return err + } + ContainersCounter.Inc(1) + s.containers[id] = &containerInfo{ + container: container, + } + logrus.WithField("id", id).Debug("containerd: container restored") + for _, p := range processes { + if err := s.monitorProcess(p); err != nil { + return err + } + } + } + return nil } diff --git a/supervisor/supervisor_linux.go b/supervisor/supervisor_linux.go deleted file mode 100644 index ec75542e4..000000000 --- a/supervisor/supervisor_linux.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build libcontainer - -package supervisor - -import ( - "github.com/docker/containerd/linux" - "github.com/docker/containerd/runtime" -) - -func newRuntime(stateDir string) (runtime.Runtime, error) { - return linux.NewRuntime(stateDir) -} diff --git a/supervisor/supervisor_runc.go b/supervisor/supervisor_runc.go deleted file mode 100644 index a0b7b49d0..000000000 --- a/supervisor/supervisor_runc.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build runc - -package supervisor - -import ( - "github.com/docker/containerd/runc" - "github.com/docker/containerd/runtime" -) - -func newRuntime(stateDir string) (runtime.Runtime, error) { - return runc.NewRuntime(stateDir) -} diff --git a/supervisor/supervisor_unsupported.go b/supervisor/supervisor_unsupported.go deleted file mode 100644 index 2b548930f..000000000 --- a/supervisor/supervisor_unsupported.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !libcontainer,!runc - -package supervisor - -import ( - "errors" - - "github.com/docker/containerd/runtime" -) - -func newRuntime(stateDir string) (runtime.Runtime, error) { - return nil, errors.New("unsupported platform") -} diff --git a/supervisor/update.go b/supervisor/update.go index 5bd4ffa2c..7bc429286 100644 --- a/supervisor/update.go +++ b/supervisor/update.go @@ -27,15 +27,17 @@ func (h *UpdateEvent) Handle(e *Event) error { } } if e.Signal != nil { - // signal the pid1/main process of the container - processes, err := container.Processes() - if err != nil { - return err - } - if len(processes) == 0 { - return ErrProcessNotFound - } - return processes[0].Signal(e.Signal) + /* + // signal the pid1/main process of the container + processes, err := container.Processes() + if err != nil { + return err + } + if len(processes) == 0 { + return ErrProcessNotFound + } + return processes[0].Signal(e.Signal) + */ } return nil } diff --git a/supervisor/worker.go b/supervisor/worker.go index 4acb8d26e..6759b6959 100644 --- a/supervisor/worker.go +++ b/supervisor/worker.go @@ -38,26 +38,24 @@ type worker struct { func (w *worker) Start() { defer w.wg.Done() for t := range w.s.tasks { - started := time.Now() - l, err := w.s.copyIO(t.Stdin, t.Stdout, t.Stderr, t.IO) - if err != nil { - evt := NewEvent(DeleteEventType) - evt.ID = t.Container.ID() - w.s.SendEvent(evt) - t.Err <- err - continue - } - w.s.containers[t.Container.ID()].copier = l + var ( + err error + process runtime.Process + started = time.Now() + ) if t.Checkpoint != "" { - if err := t.Container.Restore(t.Checkpoint); err != nil { - evt := NewEvent(DeleteEventType) - evt.ID = t.Container.ID() - w.s.SendEvent(evt) - t.Err <- err - continue - } + /* + if err := t.Container.Restore(t.Checkpoint); err != nil { + evt := NewEvent(DeleteEventType) + evt.ID = t.Container.ID() + w.s.SendEvent(evt) + t.Err <- err + continue + } + */ } else { - if err := t.Container.Start(); err != nil { + process, err = t.Container.Start() + if err != nil { evt := NewEvent(DeleteEventType) evt.ID = t.Container.ID() w.s.SendEvent(evt) @@ -65,22 +63,25 @@ func (w *worker) Start() { continue } } - pid, err := t.Container.Pid() - if err != nil { - logrus.WithField("error", err).Error("containerd: get container main pid") - } - if w.s.notifier != nil { - n, err := t.Container.OOM() - if err != nil { - logrus.WithField("error", err).Error("containerd: notify OOM events") - } else { - w.s.notifier.Add(t.Container.ID(), n) - } + /* + if w.s.notifier != nil { + n, err := t.Container.OOM() + if err != nil { + logrus.WithField("error", err).Error("containerd: notify OOM events") + } else { + w.s.notifier.Add(n, t.Container.ID()) + } + } + */ + if err := w.s.monitorProcess(process); err != nil { + logrus.WithField("error", err).Error("containerd: add process to monitor") } ContainerStartTimer.UpdateSince(started) t.Err <- nil t.StartResponse <- StartResponse{ - Pid: pid, + Stdin: process.Stdin(), + Stdout: process.Stdout(), + Stderr: process.Stderr(), } } } diff --git a/util/reaper.go b/util/reaper.go new file mode 100644 index 000000000..34b4c2f33 --- /dev/null +++ b/util/reaper.go @@ -0,0 +1,38 @@ +package util + +import ( + "syscall" + + "github.com/opencontainers/runc/libcontainer/utils" +) + +// Exit is the wait4 information from an exited process +type Exit struct { + Pid int + Status int +} + +// Reap reaps all child processes for the calling process and returns their +// exit information +func Reap() (exits []Exit, err error) { + var ( + ws syscall.WaitStatus + rus syscall.Rusage + ) + for { + pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus) + if err != nil { + if err == syscall.ECHILD { + return exits, nil + } + return exits, err + } + if pid <= 0 { + return exits, nil + } + exits = append(exits, Exit{ + Pid: pid, + Status: utils.ExitStatus(ws), + }) + } +}