From 63f89eb9540a6aa4061b559281c23e82a1544110 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 4 Aug 2020 04:51:08 -0400 Subject: [PATCH] Update server with nri injection points This allows development with container to be done for NRI without the need for custom builds. This is an experimental feature and is not enabled unless a user has a global `/etc/nri/conf.json` config setup with plugins on the system. No NRI code will be executed if this config file does not exist. Signed-off-by: Michael Crosby --- pkg/server/container_start.go | 16 +++++- pkg/server/events.go | 4 +- pkg/server/opts.go | 51 +++++++++++++++++++ pkg/server/sandbox_run.go | 18 ++++++- vendor.conf | 2 +- vendor/github.com/containerd/nri/client.go | 3 ++ .../containerd/nri/types/v1/types.go | 2 + 7 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 pkg/server/opts.go diff --git a/pkg/server/container_start.go b/pkg/server/container_start.go index c1f860d59..d00eb3d8e 100644 --- a/pkg/server/container_start.go +++ b/pkg/server/container_start.go @@ -24,6 +24,8 @@ import ( containerdio "github.com/containerd/containerd/cio" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" + "github.com/containerd/nri" + v1 "github.com/containerd/nri/types/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" @@ -107,7 +109,7 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain deferCtx, deferCancel := ctrdutil.DeferContext() defer deferCancel() // It's possible that task is deleted by event monitor. - if _, err := task.Delete(deferCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + if _, err := task.Delete(deferCtx, WithNRISandboxDelete(sandboxID), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { log.G(ctx).WithError(err).Errorf("Failed to delete containerd task %q", id) } } @@ -118,6 +120,18 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain if err != nil { return nil, errors.Wrap(err, "failed to wait for containerd task") } + nric, err := nri.New() + if err != nil { + log.G(ctx).WithError(err).Error("unable to create nri client") + } + if nric != nil { + nriSB := &nri.Sandbox{ + ID: sandboxID, + } + if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil { + return nil, errors.Wrap(err, "nri invoke") + } + } // Start containerd task. if err := task.Start(ctx); err != nil { diff --git a/pkg/server/events.go b/pkg/server/events.go index df465f22c..8f35bf0bd 100644 --- a/pkg/server/events.go +++ b/pkg/server/events.go @@ -317,7 +317,7 @@ func handleContainerExit(ctx context.Context, e *eventtypes.TaskExit, cntr conta } } else { // TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker - if _, err = task.Delete(ctx, containerd.WithProcessKill); err != nil { + if _, err = task.Delete(ctx, WithNRISandboxDelete(cntr.SandboxID), containerd.WithProcessKill); err != nil { if !errdefs.IsNotFound(err) { return errors.Wrap(err, "failed to stop container") } @@ -359,7 +359,7 @@ func handleSandboxExit(ctx context.Context, e *eventtypes.TaskExit, sb sandboxst } } else { // TODO(random-liu): [P1] This may block the loop, we may want to spawn a worker - if _, err = task.Delete(ctx, containerd.WithProcessKill); err != nil { + if _, err = task.Delete(ctx, WithNRISandboxDelete(sb.ID), containerd.WithProcessKill); err != nil { if !errdefs.IsNotFound(err) { return errors.Wrap(err, "failed to stop sandbox") } diff --git a/pkg/server/opts.go b/pkg/server/opts.go new file mode 100644 index 000000000..58520dbc3 --- /dev/null +++ b/pkg/server/opts.go @@ -0,0 +1,51 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import ( + "context" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/log" + "github.com/containerd/nri" + v1 "github.com/containerd/nri/types/v1" +) + +// WithNRISandboxDelete calls delete for a sandbox'd task +func WithNRISandboxDelete(sandboxID string) containerd.ProcessDeleteOpts { + return func(ctx context.Context, p containerd.Process) error { + task, ok := p.(containerd.Task) + if !ok { + return nil + } + nric, err := nri.New() + if err != nil { + log.G(ctx).WithError(err).Error("unable to create nri client") + return nil + } + if nric == nil { + return nil + } + sb := &nri.Sandbox{ + ID: sandboxID, + } + if _, err := nric.InvokeWithSandbox(ctx, task, v1.Delete, sb); err != nil { + log.G(ctx).WithError(err).Errorf("Failed to delete nri for %q", task.ID()) + } + return nil + } +} diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go index 1fc92adbf..e4b46e26b 100644 --- a/pkg/server/sandbox_run.go +++ b/pkg/server/sandbox_run.go @@ -27,6 +27,8 @@ import ( "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" cni "github.com/containerd/go-cni" + "github.com/containerd/nri" + v1 "github.com/containerd/nri/types/v1" "github.com/containerd/typeurl" "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" @@ -269,7 +271,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox deferCtx, deferCancel := ctrdutil.DeferContext() defer deferCancel() // Cleanup the sandbox container if an error is returned. - if _, err := task.Delete(deferCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + if _, err := task.Delete(deferCtx, WithNRISandboxDelete(id), containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { log.G(ctx).WithError(err).Errorf("Failed to delete sandbox container %q", id) } } @@ -281,6 +283,20 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, errors.Wrap(err, "failed to wait for sandbox container task") } + nric, err := nri.New() + if err != nil { + return nil, errors.Wrap(err, "unable to create nri client") + } + if nric != nil { + nriSB := &nri.Sandbox{ + ID: id, + Labels: config.Labels, + } + if _, err := nric.InvokeWithSandbox(ctx, task, v1.Create, nriSB); err != nil { + return nil, errors.Wrap(err, "nri invoke") + } + } + if err := task.Start(ctx); err != nil { return nil, errors.Wrapf(err, "failed to start sandbox container task %q", id) } diff --git a/vendor.conf b/vendor.conf index 023d69c54..d78c4a103 100644 --- a/vendor.conf +++ b/vendor.conf @@ -14,7 +14,7 @@ github.com/containerd/containerd v1.4.0 github.com/containerd/continuity efbc4488d8fe1bdc16bde3b2d2990d9b3a899165 github.com/containerd/fifo f15a3290365b9d2627d189e619ab4008e0069caf github.com/containerd/go-runc 7016d3ce2328dd2cb1192b2076ebd565c4e8df0c -github.com/containerd/nri 0072507af16308562084abb520fab15440d0b309 +github.com/containerd/nri 0afc7f031eaf9c7d9c1a381b7ab5462e89c998fc github.com/containerd/ttrpc v1.0.1 github.com/containerd/typeurl v1.0.1 github.com/coreos/go-systemd/v22 v22.1.0 diff --git a/vendor/github.com/containerd/nri/client.go b/vendor/github.com/containerd/nri/client.go index 07f43e3fc..6b6c6e409 100644 --- a/vendor/github.com/containerd/nri/client.go +++ b/vendor/github.com/containerd/nri/client.go @@ -46,6 +46,8 @@ type Client struct { type Sandbox struct { // ID of the sandbox ID string + // Labels of the sandbox + Labels map[string]string } // Invoke the ConfList of nri plugins @@ -75,6 +77,7 @@ func (c *Client) InvokeWithSandbox(ctx context.Context, task containerd.Task, st } if sandbox != nil { r.SandboxID = sandbox.ID + r.Labels = sandbox.Labels } for _, p := range c.conf.Plugins { r.Conf = p.Conf diff --git a/vendor/github.com/containerd/nri/types/v1/types.go b/vendor/github.com/containerd/nri/types/v1/types.go index 412075ee2..e0703f5a0 100644 --- a/vendor/github.com/containerd/nri/types/v1/types.go +++ b/vendor/github.com/containerd/nri/types/v1/types.go @@ -75,6 +75,8 @@ type Request struct { Pid int `json:"pid,omitempty"` // Spec generated from the OCI runtime specification Spec *Spec `json:"spec"` + // Labels of a sandbox + Labels map[string]string `json:"labels,omitempty"` // Results from previous plugins in the chain Results []*Result `json:"results,omitempty"` }