Merge pull request #3292 from crosbymichael/shim-cgroup

Add shim cgroup support for v2 runtimes

container_linux_test.go

@@ -133,10 +133,6 @@ func TestShimInCgroup(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.Close()
-	if CheckRuntime(client.runtime, "io.containerd.runc") {
-		t.Skip()
-	}
-
 	var (
 		ctx, cancel = testContext()
 		id          = t.Name()
@@ -160,12 +156,7 @@ func TestShimInCgroup(t *testing.T) {
 	}
 	defer cg.Delete()
 
-	task, err := container.NewTask(ctx, empty(), func(_ context.Context, client *Client, r *TaskInfo) error {
+	task, err := container.NewTask(ctx, empty(), WithShimCgroup(path))
-		r.Options = &runctypes.CreateOptions{
-			ShimCgroup: path,
-		}
-		return nil
-	})
 	if err != nil {
 		t.Fatal(err)
 	}

runtime/v2/binary.go

@@ -30,6 +30,7 @@ import (
 	client "github.com/containerd/containerd/runtime/v2/shim"
 	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/containerd/ttrpc"
+	"github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -52,7 +53,7 @@ type binary struct {
 	rtTasks           *runtime.TaskList
 }
 
-func (b *binary) Start(ctx context.Context, onClose func()) (_ *shim, err error) {
+func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) {
 	args := []string{"-id", b.bundle.ID}
 	if logrus.GetLevel() == logrus.DebugLevel {
 		args = append(args, "-debug")
@@ -64,6 +65,7 @@ func (b *binary) Start(ctx context.Context, onClose func()) (_ *shim, err error)
 		b.runtime,
 		b.containerdAddress,
 		b.bundle.Path,
+		opts,
 		args...,
 	)
 	if err != nil {
@@ -126,6 +128,7 @@ func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
 		b.runtime,
 		b.containerdAddress,
 		bundlePath,
+		nil,
 		"-id", b.bundle.ID,
 		"-bundle", b.bundle.Path,
 		"delete")

runtime/v2/manager.go

@@ -126,8 +126,13 @@ func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.Create
 			bundle.Delete()
 		}
 	}()
+	topts := opts.TaskOptions
+	if topts == nil {
+		topts = opts.RuntimeOptions
+	}
+
 	b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.events, m.tasks)
-	shim, err := b.Start(ctx, func() {
+	shim, err := b.Start(ctx, topts, func() {
 		log.G(ctx).WithField("id", id).Info("shim disconnected")
 		_, err := m.tasks.Get(ctx, id)
 		if err != nil {

runtime/v2/runc/v1/service.go

@@ -44,6 +44,8 @@ import (
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	runcC "github.com/containerd/go-runc"
 	"github.com/containerd/typeurl"
+	"github.com/gogo/protobuf/proto"
+	"github.com/gogo/protobuf/types"
 	ptypes "github.com/gogo/protobuf/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -163,6 +165,31 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container
 	if err := shim.WriteAddress("address", address); err != nil {
 		return "", err
 	}
+	if data, err := ioutil.ReadAll(os.Stdin); err == nil {
+		if len(data) > 0 {
+			var any types.Any
+			if err := proto.Unmarshal(data, &any); err != nil {
+				return "", err
+			}
+			v, err := typeurl.UnmarshalAny(&any)
+			if err != nil {
+				return "", err
+			}
+			if opts, ok := v.(*options.Options); ok {
+				if opts.ShimCgroup != "" {
+					cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(opts.ShimCgroup))
+					if err != nil {
+						return "", errors.Wrapf(err, "failed to load cgroup %s", opts.ShimCgroup)
+					}
+					if err := cg.Add(cgroups.Process{
+						Pid: cmd.Process.Pid,
+					}); err != nil {
+						return "", errors.Wrapf(err, "failed to join cgroup %s", opts.ShimCgroup)
+					}
+				}
+			}
+		}
+	}
 	if err := shim.SetScore(cmd.Process.Pid); err != nil {
 		return "", errors.Wrap(err, "failed to set OOM Score on shim")
 	}

runtime/v2/runc/v2/service.go

@@ -45,6 +45,8 @@ import (
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 	runcC "github.com/containerd/go-runc"
 	"github.com/containerd/typeurl"
+	"github.com/gogo/protobuf/proto"
+	"github.com/gogo/protobuf/types"
 	ptypes "github.com/gogo/protobuf/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -206,6 +208,31 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container
 	if err := shim.WriteAddress("address", address); err != nil {
 		return "", err
 	}
+	if data, err := ioutil.ReadAll(os.Stdin); err == nil {
+		if len(data) > 0 {
+			var any types.Any
+			if err := proto.Unmarshal(data, &any); err != nil {
+				return "", err
+			}
+			v, err := typeurl.UnmarshalAny(&any)
+			if err != nil {
+				return "", err
+			}
+			if opts, ok := v.(*options.Options); ok {
+				if opts.ShimCgroup != "" {
+					cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(opts.ShimCgroup))
+					if err != nil {
+						return "", errors.Wrapf(err, "failed to load cgroup %s", opts.ShimCgroup)
+					}
+					if err := cg.Add(cgroups.Process{
+						Pid: cmd.Process.Pid,
+					}); err != nil {
+						return "", errors.Wrapf(err, "failed to join cgroup %s", opts.ShimCgroup)
+					}
+				}
+			}
+		}
+	}
 	if err := shim.SetScore(cmd.Process.Pid); err != nil {
 		return "", errors.Wrap(err, "failed to set OOM Score on shim")
 	}

runtime/v2/shim/util.go

@@ -17,6 +17,7 @@
 package shim
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -29,13 +30,15 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/namespaces"
+	"github.com/gogo/protobuf/proto"
+	"github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
 )
 
 var runtimePaths sync.Map
 
 // Command returns the shim command with the provided args and configuration
-func Command(ctx context.Context, runtime, containerdAddress, path string, cmdArgs ...string) (*exec.Cmd, error) {
+func Command(ctx context.Context, runtime, containerdAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) {
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return nil, err
@@ -94,6 +97,13 @@ func Command(ctx context.Context, runtime, containerdAddress, path string, cmdAr
 	cmd.Dir = path
 	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
 	cmd.SysProcAttr = getSysProcAttr()
+	if opts != nil {
+		d, err := proto.Marshal(opts)
+		if err != nil {
+			return nil, err
+		}
+		cmd.Stdin = bytes.NewReader(d)
+	}
 	return cmd, nil
 }
 

task_opts_unix.go

@@ -77,3 +77,29 @@ func WithNoPivotRoot(_ context.Context, _ *Client, ti *TaskInfo) error {
 	}
 	return nil
 }
+
+// WithShimCgroup sets the existing cgroup for the shim
+func WithShimCgroup(path string) NewTaskOpts {
+	return func(ctx context.Context, c *Client, ti *TaskInfo) error {
+		if CheckRuntime(ti.Runtime(), "io.containerd.runc") {
+			if ti.Options == nil {
+				ti.Options = &options.Options{}
+			}
+			opts, ok := ti.Options.(*options.Options)
+			if !ok {
+				return errors.New("invalid v2 shim create options format")
+			}
+			opts.ShimCgroup = path
+		} else {
+			if ti.Options == nil {
+				ti.Options = &runctypes.CreateOptions{}
+			}
+			opts, ok := ti.Options.(*runctypes.CreateOptions)
+			if !ok {
+				return errors.New("could not cast TaskInfo Options to CreateOptions")
+			}
+			opts.ShimCgroup = path
+		}
+		return nil
+	}
+}