From 90c6c1af43dc0fc3cb582d38f5f5c75586f4d132 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 17 May 2019 21:02:23 +0000 Subject: [PATCH 1/2] Pass options on shim create for v2 Signed-off-by: Michael Crosby --- runtime/v2/binary.go | 5 ++++- runtime/v2/manager.go | 7 ++++++- runtime/v2/shim/util.go | 12 +++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/runtime/v2/binary.go b/runtime/v2/binary.go index a2c6e92b9..2087509ba 100644 --- a/runtime/v2/binary.go +++ b/runtime/v2/binary.go @@ -30,6 +30,7 @@ import ( client "github.com/containerd/containerd/runtime/v2/shim" "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" + "github.com/gogo/protobuf/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -52,7 +53,7 @@ type binary struct { rtTasks *runtime.TaskList } -func (b *binary) Start(ctx context.Context, onClose func()) (_ *shim, err error) { +func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) { args := []string{"-id", b.bundle.ID} if logrus.GetLevel() == logrus.DebugLevel { args = append(args, "-debug") @@ -64,6 +65,7 @@ func (b *binary) Start(ctx context.Context, onClose func()) (_ *shim, err error) b.runtime, b.containerdAddress, b.bundle.Path, + opts, args..., ) if err != nil { @@ -126,6 +128,7 @@ func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) { b.runtime, b.containerdAddress, bundlePath, + nil, "-id", b.bundle.ID, "-bundle", b.bundle.Path, "delete") diff --git a/runtime/v2/manager.go b/runtime/v2/manager.go index 19daefdba..059ae0c80 100644 --- a/runtime/v2/manager.go +++ b/runtime/v2/manager.go @@ -126,8 +126,13 @@ func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.Create bundle.Delete() } }() + topts := opts.TaskOptions + if topts == nil { + topts = opts.RuntimeOptions + } + b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.events, m.tasks) - shim, err := b.Start(ctx, func() { + shim, err := b.Start(ctx, topts, func() { log.G(ctx).WithField("id", id).Info("shim disconnected") _, err := m.tasks.Get(ctx, id) if err != nil { diff --git a/runtime/v2/shim/util.go b/runtime/v2/shim/util.go index 2e34444a8..48e1e66d9 100644 --- a/runtime/v2/shim/util.go +++ b/runtime/v2/shim/util.go @@ -17,6 +17,7 @@ package shim import ( + "bytes" "context" "fmt" "io/ioutil" @@ -29,13 +30,15 @@ import ( "time" "github.com/containerd/containerd/namespaces" + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" "github.com/pkg/errors" ) var runtimePaths sync.Map // Command returns the shim command with the provided args and configuration -func Command(ctx context.Context, runtime, containerdAddress, path string, cmdArgs ...string) (*exec.Cmd, error) { +func Command(ctx context.Context, runtime, containerdAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { return nil, err @@ -94,6 +97,13 @@ func Command(ctx context.Context, runtime, containerdAddress, path string, cmdAr cmd.Dir = path cmd.Env = append(os.Environ(), "GOMAXPROCS=2") cmd.SysProcAttr = getSysProcAttr() + if opts != nil { + d, err := proto.Marshal(opts) + if err != nil { + return nil, err + } + cmd.Stdin = bytes.NewReader(d) + } return cmd, nil } From fe6a2b03ede2cff4529f4d7f29827a155f13332c Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 20 May 2019 15:10:21 +0000 Subject: [PATCH 2/2] Add shim cgroup support for v2 runtimes Closes #3198 Signed-off-by: Michael Crosby --- container_linux_test.go | 11 +---------- runtime/v2/runc/v1/service.go | 27 +++++++++++++++++++++++++++ runtime/v2/runc/v2/service.go | 27 +++++++++++++++++++++++++++ task_opts_unix.go | 26 ++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/container_linux_test.go b/container_linux_test.go index 8cc9555dd..ac1c8c47c 100644 --- a/container_linux_test.go +++ b/container_linux_test.go @@ -133,10 +133,6 @@ func TestShimInCgroup(t *testing.T) { t.Fatal(err) } defer client.Close() - if CheckRuntime(client.runtime, "io.containerd.runc") { - t.Skip() - } - var ( ctx, cancel = testContext() id = t.Name() @@ -160,12 +156,7 @@ func TestShimInCgroup(t *testing.T) { } defer cg.Delete() - task, err := container.NewTask(ctx, empty(), func(_ context.Context, client *Client, r *TaskInfo) error { - r.Options = &runctypes.CreateOptions{ - ShimCgroup: path, - } - return nil - }) + task, err := container.NewTask(ctx, empty(), WithShimCgroup(path)) if err != nil { t.Fatal(err) } diff --git a/runtime/v2/runc/v1/service.go b/runtime/v2/runc/v1/service.go index 269d26471..2125b8ae7 100644 --- a/runtime/v2/runc/v1/service.go +++ b/runtime/v2/runc/v1/service.go @@ -44,6 +44,8 @@ import ( taskAPI "github.com/containerd/containerd/runtime/v2/task" runcC "github.com/containerd/go-runc" "github.com/containerd/typeurl" + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" ptypes "github.com/gogo/protobuf/types" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -163,6 +165,31 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container if err := shim.WriteAddress("address", address); err != nil { return "", err } + if data, err := ioutil.ReadAll(os.Stdin); err == nil { + if len(data) > 0 { + var any types.Any + if err := proto.Unmarshal(data, &any); err != nil { + return "", err + } + v, err := typeurl.UnmarshalAny(&any) + if err != nil { + return "", err + } + if opts, ok := v.(*options.Options); ok { + if opts.ShimCgroup != "" { + cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(opts.ShimCgroup)) + if err != nil { + return "", errors.Wrapf(err, "failed to load cgroup %s", opts.ShimCgroup) + } + if err := cg.Add(cgroups.Process{ + Pid: cmd.Process.Pid, + }); err != nil { + return "", errors.Wrapf(err, "failed to join cgroup %s", opts.ShimCgroup) + } + } + } + } + } if err := shim.SetScore(cmd.Process.Pid); err != nil { return "", errors.Wrap(err, "failed to set OOM Score on shim") } diff --git a/runtime/v2/runc/v2/service.go b/runtime/v2/runc/v2/service.go index 9fc2d1b9d..4623a78ed 100644 --- a/runtime/v2/runc/v2/service.go +++ b/runtime/v2/runc/v2/service.go @@ -45,6 +45,8 @@ import ( taskAPI "github.com/containerd/containerd/runtime/v2/task" runcC "github.com/containerd/go-runc" "github.com/containerd/typeurl" + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" ptypes "github.com/gogo/protobuf/types" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -206,6 +208,31 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container if err := shim.WriteAddress("address", address); err != nil { return "", err } + if data, err := ioutil.ReadAll(os.Stdin); err == nil { + if len(data) > 0 { + var any types.Any + if err := proto.Unmarshal(data, &any); err != nil { + return "", err + } + v, err := typeurl.UnmarshalAny(&any) + if err != nil { + return "", err + } + if opts, ok := v.(*options.Options); ok { + if opts.ShimCgroup != "" { + cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(opts.ShimCgroup)) + if err != nil { + return "", errors.Wrapf(err, "failed to load cgroup %s", opts.ShimCgroup) + } + if err := cg.Add(cgroups.Process{ + Pid: cmd.Process.Pid, + }); err != nil { + return "", errors.Wrapf(err, "failed to join cgroup %s", opts.ShimCgroup) + } + } + } + } + } if err := shim.SetScore(cmd.Process.Pid); err != nil { return "", errors.Wrap(err, "failed to set OOM Score on shim") } diff --git a/task_opts_unix.go b/task_opts_unix.go index d3b51a76d..8b498d47e 100644 --- a/task_opts_unix.go +++ b/task_opts_unix.go @@ -77,3 +77,29 @@ func WithNoPivotRoot(_ context.Context, _ *Client, ti *TaskInfo) error { } return nil } + +// WithShimCgroup sets the existing cgroup for the shim +func WithShimCgroup(path string) NewTaskOpts { + return func(ctx context.Context, c *Client, ti *TaskInfo) error { + if CheckRuntime(ti.Runtime(), "io.containerd.runc") { + if ti.Options == nil { + ti.Options = &options.Options{} + } + opts, ok := ti.Options.(*options.Options) + if !ok { + return errors.New("invalid v2 shim create options format") + } + opts.ShimCgroup = path + } else { + if ti.Options == nil { + ti.Options = &runctypes.CreateOptions{} + } + opts, ok := ti.Options.(*runctypes.CreateOptions) + if !ok { + return errors.New("could not cast TaskInfo Options to CreateOptions") + } + opts.ShimCgroup = path + } + return nil + } +}