github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add ShimCgroup path for placing shim in cgroup

Browse Source 
Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby 2017-07-06 12:11:37 -07:00
parent d4349eff39
commit 98a86c4d38
8 changed files with 227 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
container_linux_test.go
View File

@ -3,10 +3,12 @@
package containerd
import (
"context"
"syscall"
"testing"
"github.com/containerd/cgroups"
"github.com/containerd/containerd/linux/runcopts"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
@ -98,3 +100,78 @@ func TestContainerUpdate(t *testing.T) {
<-statusC
}
func TestShimInCgroup(t *testing.T) {
client, err := newClient(t, address)
if err != nil {
t.Fatal(err)
}
defer client.Close()
var (
ctx, cancel = testContext()
id = t.Name()
)
defer cancel()
image, err := client.GetImage(ctx, testImage)
if err != nil {
t.Error(err)
return
}
spec, err := GenerateSpec(WithImageConfig(ctx, image), WithProcessArgs("sleep", "30"))
if err != nil {
t.Error(err)
return
}
container, err := client.NewContainer(ctx, id, WithSpec(spec), WithNewSnapshot(id, image))
if err != nil {
t.Error(err)
return
}
defer container.Delete(ctx, WithSnapshotCleanup)
// create a cgroup for the shim to use
path := "/containerd/shim"
cg, err := cgroups.New(cgroups.V1, cgroups.StaticPath(path), &specs.LinuxResources{})
if err != nil {
t.Error(err)
return
}
defer cg.Delete()
task, err := container.NewTask(ctx, empty(), func(_ context.Context, client *Client, r *TaskInfo) error {
r.Options = &runcopts.CreateOptions{
ShimCgroup: path,
}
return nil
})
if err != nil {
t.Error(err)
return
}
defer task.Delete(ctx)
statusC := make(chan uint32, 1)
go func() {
status, err := task.Wait(ctx)
if err != nil {
t.Error(err)
}
statusC <- status
}()
// check to see if the shim is inside the cgroup
processes, err := cg.Processes(cgroups.Devices, false)
if err != nil {
t.Error(err)
return
}
if len(processes) == 0 {
t.Errorf("created cgroup should have atleast one process inside: %d", len(processes))
}
if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
t.Error(err)
return
}
<-statusC
}

14
linux/bundle.go
View File

@ -9,7 +9,10 @@ import (
"os"
"path/filepath"
"github.com/containerd/containerd/linux/runcopts"
client "github.com/containerd/containerd/linux/shim"
"github.com/containerd/containerd/runtime"
"github.com/containerd/containerd/typeurl"
)
func loadBundle(path, namespace string) *bundle {
@ -56,15 +59,24 @@ type bundle struct {
}
// NewShim connects to the shim managing the bundle and tasks
func (b *bundle) NewShim(ctx context.Context, binary, grpcAddress string, remote, debug bool) (*client.Client, error) {
func (b *bundle) NewShim(ctx context.Context, binary, grpcAddress string, remote, debug bool, createOpts runtime.CreateOpts) (*client.Client, error) {
opt := client.WithStart(binary, grpcAddress, debug)
if !remote {
opt = client.WithLocal
}
var options runcopts.CreateOptions
if createOpts.Options != nil {
v, err := typeurl.UnmarshalAny(createOpts.Options)
if err != nil {
return nil, err
}
options = *v.(*runcopts.CreateOptions)
}
return client.New(ctx, client.Config{
Address: b.shimAddress(),
Path: b.path,
Namespace: b.namespace,
CgroupPath: options.ShimCgroup,
}, opt)
}

98
linux/runcopts/runc.pb.go
View File

@ -54,6 +54,7 @@ type CreateOptions struct {
EmptyNamespaces []string `protobuf:"bytes,6,rep,name=empty_namespaces,json=emptyNamespaces" json:"empty_namespaces,omitempty"`
CgroupsMode string `protobuf:"bytes,7,opt,name=cgroups_mode,json=cgroupsMode,proto3" json:"cgroups_mode,omitempty"`
NoNewKeyring bool `protobuf:"varint,8,opt,name=no_new_keyring,json=noNewKeyring,proto3" json:"no_new_keyring,omitempty"`
ShimCgroup string `protobuf:"bytes,9,opt,name=shim_cgroup,json=shimCgroup,proto3" json:"shim_cgroup,omitempty"`
}
func (m *CreateOptions) Reset() { *m = CreateOptions{} }
@ -205,6 +206,12 @@ func (m *CreateOptions) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if len(m.ShimCgroup) > 0 {
dAtA[i] = 0x4a
i++
i = encodeVarintRunc(dAtA, i, uint64(len(m.ShimCgroup)))
i += copy(dAtA[i:], m.ShimCgroup)
}
return i, nil
}
@ -369,6 +376,10 @@ func (m *CreateOptions) Size() (n int) {
if m.NoNewKeyring {
n += 2
}
l = len(m.ShimCgroup)
if l > 0 {
n += 1 + l + sovRunc(uint64(l))
}
return n
}
@ -440,6 +451,7 @@ func (this *CreateOptions) String() string {
`EmptyNamespaces:` + fmt.Sprintf("%v", this.EmptyNamespaces) + `,`,
`CgroupsMode:` + fmt.Sprintf("%v", this.CgroupsMode) + `,`,
`NoNewKeyring:` + fmt.Sprintf("%v", this.NoNewKeyring) + `,`,
`ShimCgroup:` + fmt.Sprintf("%v", this.ShimCgroup) + `,`,
`}`,
}, "")
return s
@ -783,6 +795,35 @@ func (m *CreateOptions) Unmarshal(dAtA []byte) error {
}
}
m.NoNewKeyring = bool(v != 0)
case 9:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ShimCgroup", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRunc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthRunc
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ShimCgroup = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipRunc(dAtA[iNdEx:])
@ -1122,32 +1163,33 @@ func init() {
}
var fileDescriptorRunc = []byte{
// 428 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x93, 0x31, 0x6f, 0xd4, 0x4c,
0x10, 0x86, 0xe3, 0x24, 0x5f, 0xe2, 0xdb, 0xcb, 0xe5, 0x83, 0x85, 0x48, 0x26, 0x08, 0x2b, 0x9c,
0x40, 0x0a, 0xcd, 0x9d, 0x04, 0x0d, 0x82, 0x8e, 0x2b, 0x81, 0x10, 0x0c, 0x34, 0x34, 0x2b, 0x67,
0x6f, 0xf0, 0xad, 0xce, 0x9e, 0x59, 0xed, 0x8e, 0x89, 0xaf, 0xe3, 0x2f, 0xf1, 0x2f, 0x52, 0x52,
0x52, 0x92, 0xfb, 0x23, 0x20, 0xaf, 0xe3, 0x40, 0x4b, 0x4b, 0xf7, 0xce, 0x33, 0x8f, 0xc6, 0xd2,
0x2b, 0xaf, 0x78, 0x56, 0x18, 0x5e, 0xd4, 0x67, 0x13, 0x4d, 0xd5, 0x54, 0x13, 0x72, 0x6e, 0x10,
0xdc, 0xfc, 0xcf, 0x58, 0x1a, 0xac, 0x9b, 0xa9, 0xab, 0x51, 0x93, 0x65, 0x1f, 0xc2, 0xc4, 0x3a,
0x62, 0x92, 0x07, 0xbf, 0xad, 0x49, 0xb0, 0x26, 0xed, 0xf2, 0xf0, 0x76, 0x41, 0x05, 0x05, 0x63,
0xda, 0xa6, 0x4e, 0x1e, 0xbf, 0x15, 0xc3, 0xac, 0x46, 0xfd, 0xc6, 0xb2, 0x21, 0xf4, 0xf2, 0xae,
0x18, 0x68, 0x67, 0x6a, 0x65, 0x73, 0x5e, 0x24, 0xd1, 0x51, 0x74, 0x3c, 0xc8, 0xe2, 0x16, 0x9c,
0xe6, 0xbc, 0x90, 0x0f, 0xc5, 0xbe, 0x5f, 0x79, 0x86, 0x6a, 0xae, 0x74, 0xe1, 0xa8, 0xb6, 0xc9,
0x66, 0x30, 0x46, 0x57, 0x74, 0x16, 0xe0, 0xf8, 0xeb, 0xa6, 0x18, 0xcd, 0x1c, 0xe4, 0x0c, 0xfd,
0xd5, 0xb1, 0x18, 0x21, 0x29, 0x6b, 0x3e, 0x13, 0x2b, 0x47, 0xc4, 0xe1, 0x72, 0x9c, 0x0d, 0x91,
0x4e, 0x5b, 0x96, 0x11, 0xb1, 0xbc, 0x23, 0x62, 0xb2, 0x80, 0x8a, 0x75, 0x77, 0x36, 0xce, 0x76,
0xdb, 0xf9, 0xbd, 0xb6, 0xf2, 0xb1, 0x38, 0x80, 0x86, 0xc1, 0x61, 0x5e, 0xaa, 0x1a, 0x4d, 0xa3,
0x3c, 0xe9, 0x25, 0xb0, 0x4f, 0xb6, 0x82, 0x77, 0xab, 0x5f, 0x7e, 0x40, 0xd3, 0xbc, 0xeb, 0x56,
0xf2, 0x50, 0xc4, 0x0c, 0xae, 0x32, 0x98, 0x97, 0xc9, 0x76, 0xd0, 0xae, 0x67, 0x79, 0x4f, 0x88,
0x4f, 0xa6, 0x04, 0x55, 0x92, 0x5e, 0xfa, 0xe4, 0xbf, 0xb0, 0x1d, 0xb4, 0xe4, 0x55, 0x0b, 0xe4,
0x23, 0x71, 0x03, 0x2a, 0xcb, 0x2b, 0x85, 0x79, 0x05, 0xde, 0xe6, 0x1a, 0x7c, 0xb2, 0x73, 0xb4,
0x75, 0x3c, 0xc8, 0xfe, 0x0f, 0xfc, 0xe4, 0x1a, 0xcb, 0xfb, 0x62, 0xaf, 0x6b, 0xc2, 0xab, 0x8a,
0xe6, 0x90, 0xec, 0x86, 0x3e, 0x86, 0x57, 0xec, 0x35, 0xcd, 0x41, 0x3e, 0x10, 0xfb, 0x48, 0x0a,
0xe1, 0x5c, 0x2d, 0x61, 0xe5, 0x0c, 0x16, 0x49, 0x1c, 0x3e, 0xb8, 0x87, 0x74, 0x02, 0xe7, 0x2f,
0x3b, 0x36, 0xfe, 0x19, 0x89, 0x9b, 0xb3, 0x05, 0xe8, 0xa5, 0x25, 0x83, 0xdc, 0xf7, 0x26, 0xc5,
0x36, 0x34, 0xa6, 0xaf, 0x2b, 0xe4, 0x7f, 0xb5, 0xa7, 0x17, 0xd9, 0xc5, 0x65, 0xba, 0xf1, 0xfd,
0x32, 0xdd, 0xf8, 0xb2, 0x4e, 0xa3, 0x8b, 0x75, 0x1a, 0x7d, 0x5b, 0xa7, 0xd1, 0x8f, 0x75, 0x1a,
0x7d, 0x7c, 0xfa, 0x97, 0x6f, 0xe1, 0x79, 0x1f, 0xce, 0x76, 0xc2, 0x3f, 0xfe, 0xe4, 0x57, 0x00,
0x00, 0x00, 0xff, 0xff, 0x9d, 0xd8, 0x49, 0xfe, 0x4e, 0x03, 0x00, 0x00,
// 438 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x93, 0xb1, 0x6f, 0xd4, 0x30,
0x18, 0xc5, 0x9b, 0xb6, 0xb4, 0x89, 0xaf, 0x57, 0xc0, 0x50, 0x29, 0x14, 0x11, 0xca, 0x09, 0xa4,
0xb2, 0xdc, 0x49, 0xb0, 0x20, 0xd8, 0xb8, 0x11, 0x28, 0x25, 0xc0, 0xc2, 0x62, 0xa5, 0xbe, 0x8f,
0xc4, 0xba, 0xe4, 0xfb, 0x2c, 0xdb, 0xa1, 0xb9, 0x8d, 0x3f, 0xaf, 0x23, 0x62, 0x62, 0xa4, 0xf9,
0x47, 0x40, 0x71, 0x2e, 0x85, 0x95, 0x95, 0xed, 0xf9, 0xf7, 0x9e, 0x9e, 0xa5, 0x27, 0x7d, 0xec,
0x79, 0xae, 0x5c, 0x51, 0x9f, 0x4d, 0x25, 0x55, 0x33, 0x49, 0xe8, 0x32, 0x85, 0x60, 0x16, 0x7f,
0xcb, 0x52, 0x61, 0xdd, 0xcc, 0x4c, 0x8d, 0x92, 0xb4, 0xb3, 0x5e, 0x4c, 0xb5, 0x21, 0x47, 0xfc,
0xe0, 0x4f, 0x6a, 0xea, 0x53, 0xd3, 0xce, 0x3c, 0xbc, 0x9d, 0x53, 0x4e, 0x3e, 0x31, 0xeb, 0x54,
0x1f, 0x9e, 0xbc, 0x63, 0xa3, 0xb4, 0x46, 0xf9, 0x56, 0x3b, 0x45, 0x68, 0xf9, 0x5d, 0x16, 0x49,
0xa3, 0x6a, 0xa1, 0x33, 0x57, 0xc4, 0xc1, 0x51, 0x70, 0x1c, 0xa5, 0x61, 0x07, 0x4e, 0x33, 0x57,
0xf0, 0x47, 0x6c, 0xdf, 0xae, 0xac, 0x83, 0x6a, 0x21, 0x64, 0x6e, 0xa8, 0xd6, 0xf1, 0xa6, 0x4f,
0x8c, 0xd7, 0x74, 0xee, 0xe1, 0xe4, 0xfb, 0x26, 0x1b, 0xcf, 0x0d, 0x64, 0x0e, 0x86, 0xd6, 0x09,
0x1b, 0x23, 0x09, 0xad, 0xbe, 0x90, 0x13, 0x86, 0xc8, 0xf9, 0xe6, 0x30, 0x1d, 0x21, 0x9d, 0x76,
0x2c, 0x25, 0x72, 0xfc, 0x0e, 0x0b, 0x49, 0x03, 0x0a, 0x27, 0xfb, 0xda, 0x30, 0xdd, 0xed, 0xde,
0x1f, 0xa4, 0xe6, 0x4f, 0xd8, 0x01, 0x34, 0x0e, 0x0c, 0x66, 0xa5, 0xa8, 0x51, 0x35, 0xc2, 0x92,
0x5c, 0x82, 0xb3, 0xf1, 0x96, 0xcf, 0xdd, 0x1a, 0xcc, 0x8f, 0xa8, 0x9a, 0xf7, 0xbd, 0xc5, 0x0f,
0x59, 0xe8, 0xc0, 0x54, 0x0a, 0xb3, 0x32, 0xde, 0xf6, 0xb1, 0xab, 0x37, 0xbf, 0xc7, 0xd8, 0x67,
0x55, 0x82, 0x28, 0x49, 0x2e, 0x6d, 0x7c, 0xcd, 0xbb, 0x51, 0x47, 0x5e, 0x77, 0x80, 0x3f, 0x66,
0x37, 0xa0, 0xd2, 0x6e, 0x25, 0x30, 0xab, 0xc0, 0xea, 0x4c, 0x82, 0x8d, 0x77, 0x8e, 0xb6, 0x8e,
0xa3, 0xf4, 0xba, 0xe7, 0x27, 0x57, 0x98, 0x3f, 0x60, 0x7b, 0xfd, 0x12, 0x56, 0x54, 0xb4, 0x80,
0x78, 0xd7, 0xef, 0x31, 0x5a, 0xb3, 0x37, 0xb4, 0x00, 0xfe, 0x90, 0xed, 0x23, 0x09, 0x84, 0x73,
0xb1, 0x84, 0x95, 0x51, 0x98, 0xc7, 0xa1, 0xff, 0x70, 0x0f, 0xe9, 0x04, 0xce, 0x5f, 0xf5, 0x8c,
0xdf, 0x67, 0x23, 0x5b, 0xa8, 0x6a, 0xd8, 0x35, 0xf2, 0x3d, 0xac, 0x43, 0xeb, 0x51, 0x7f, 0x05,
0xec, 0xe6, 0xbc, 0x00, 0xb9, 0xd4, 0xa4, 0xd0, 0x0d, 0xc3, 0x72, 0xb6, 0x0d, 0x8d, 0x1a, 0xf6,
0xf4, 0xfa, 0x7f, 0x1d, 0xf2, 0x65, 0x7a, 0x71, 0x99, 0x6c, 0xfc, 0xb8, 0x4c, 0x36, 0xbe, 0xb6,
0x49, 0x70, 0xd1, 0x26, 0xc1, 0xb7, 0x36, 0x09, 0x7e, 0xb6, 0x49, 0xf0, 0xe9, 0xd9, 0x3f, 0x1e,
0xcb, 0x8b, 0x41, 0x9c, 0xed, 0xf8, 0x23, 0x78, 0xfa, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x00, 0x19,
0xba, 0x8f, 0x6f, 0x03, 0x00, 0x00,
}

1
linux/runcopts/runc.proto
View File

@ -20,6 +20,7 @@ message CreateOptions {
repeated string empty_namespaces = 6;
string cgroups_mode = 7;
bool no_new_keyring = 8;
string shim_cgroup = 9;
}
message CheckpointOptions {

2
linux/runtime.go
View File

@ -139,7 +139,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
bundle.Delete()
}
}()
s, err := bundle.NewShim(ctx, r.shim, r.address, r.remote, r.shimDebug)
s, err := bundle.NewShim(ctx, r.shim, r.address, r.remote, r.shimDebug, opts)
if err != nil {
return nil, err
}

25
linux/shim/client.go
View File

@ -28,7 +28,7 @@ type ClientOpt func(context.Context, Config) (shim.ShimClient, io.Closer, error)
// WithStart executes a new shim process
func WithStart(binary, address string, debug bool) ClientOpt {
return func(ctx context.Context, config Config) (shim.ShimClient, io.Closer, error) {
return func(ctx context.Context, config Config) (_ shim.ShimClient, _ io.Closer, err error) {
socket, err := newSocket(config)
if err != nil {
return nil, nil, err
@ -44,16 +44,36 @@ func WithStart(binary, address string, debug bool) ClientOpt {
if err := reaper.Default.Start(cmd); err != nil {
return nil, nil, errors.Wrapf(err, "failed to start shim")
}
defer func() {
if err != nil {
terminate(cmd)
}
}()
log.G(ctx).WithFields(logrus.Fields{
"pid": cmd.Process.Pid,
"address": config.Address,
"debug": debug,
}).Infof("shim %s started", binary)
// set shim in cgroup if it is provided
if config.CgroupPath != "" {
if err := setCgroup(ctx, config, cmd); err != nil {
return nil, nil, err
}
}
if err = sys.SetOOMScore(cmd.Process.Pid, sys.OOMScoreMaxKillable); err != nil {
return nil, nil, errors.Wrap(err, "failed to set OOM Score on shim")
}
return WithConnect(ctx, config)
c, clo, err := WithConnect(ctx, config)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to connect")
}
return c, clo, nil
}
}
func terminate(cmd *exec.Cmd) {
cmd.Process.Kill()
reaper.Default.Wait(cmd)
}
func newCommand(binary, address string, debug bool, config Config, socket *os.File) *exec.Cmd {
@ -141,6 +161,7 @@ type Config struct {
Address string
Path string
Namespace string
CgroupPath string
}
// New returns a new shim client

28
linux/shim/client_linux.go
View File

@ -2,9 +2,35 @@
package shim
import "syscall"
import (
"context"
"os/exec"
"syscall"
"github.com/containerd/cgroups"
"github.com/containerd/containerd/log"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
var atter = syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWNS,
Setpgid: true,
}
func setCgroup(ctx context.Context, config Config, cmd *exec.Cmd) error {
cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(config.CgroupPath))
if err != nil {
return errors.Wrapf(err, "failed to load cgroup %s", config.CgroupPath)
}
if err := cg.Add(cgroups.Process{
Pid: cmd.Process.Pid,
}); err != nil {
return errors.Wrapf(err, "failed to join cgroup %s", config.CgroupPath)
}
log.G(ctx).WithFields(logrus.Fields{
"pid": cmd.Process.Pid,
"address": config.Address,
}).Infof("shim placed in cgroup %s", config.CgroupPath)
return nil
}

10
linux/shim/client_unix.go
View File

@ -2,8 +2,16 @@
package shim
import "syscall"
import (
"context"
"os/exec"
"syscall"
)
var atter = syscall.SysProcAttr{
Setpgid: true,
}
func setCgroup(ctx context.Context, config Config, cmd *exec.Cmd) error {
return nil
}