From e1731d2e5ec2ce9be7d4157eb897808451d4543d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 6 Mar 2017 16:26:19 -0800 Subject: [PATCH 1/8] Add linux runtime config Signed-off-by: Michael Crosby --- linux/runtime.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/linux/runtime.go b/linux/runtime.go index d063db2a3..9901e8503 100644 --- a/linux/runtime.go +++ b/linux/runtime.go @@ -22,23 +22,35 @@ import ( const ( runtimeName = "linux" configFilename = "config.json" + defaultRuntime = "runc" ) func init() { plugin.Register(runtimeName, &plugin.Registration{ - Type: plugin.RuntimePlugin, - Init: New, + Type: plugin.RuntimePlugin, + Init: New, + Config: &Config{}, }) } -func New(ic *plugin.InitContext) (interface{}, error) { +type Config struct { + // Runtime is a path or name of an OCI runtime used by the shim + Runtime string `toml:"runtime"` +} + +func New(ic *containerd.InitContext) (interface{}, error) { path := filepath.Join(ic.State, runtimeName) if err := os.MkdirAll(path, 0700); err != nil { return nil, err } + cfg := ic.Config.(*Config) + if cfg.Runtime == "" { + cfg.Runtime = defaultRuntime + } c, cancel := context.WithCancel(ic.Context) return &Runtime{ root: path, + runtime: cfg.Runtime, events: make(chan *containerd.Event, 2048), eventsContext: c, eventsCancel: cancel, @@ -46,7 +58,8 @@ func New(ic *plugin.InitContext) (interface{}, error) { } type Runtime struct { - root string + root string + runtime string events chan *containerd.Event eventsContext context.Context @@ -70,7 +83,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts containerd.CreateO sopts := &shim.CreateRequest{ ID: id, Bundle: path, - Runtime: "runc", + Runtime: r.runtime, Stdin: opts.IO.Stdin, Stdout: opts.IO.Stdout, Stderr: opts.IO.Stderr, From fc45d9c1192ebdcb3e46cbf4ef561e7b7367adbe Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 8 Mar 2017 11:42:36 -0800 Subject: [PATCH 2/8] Chown grpc socket based on config Signed-off-by: Michael Crosby --- cmd/containerd/config.go | 2 ++ cmd/containerd/main.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/cmd/containerd/config.go b/cmd/containerd/config.go index 146a4a452..996481321 100644 --- a/cmd/containerd/config.go +++ b/cmd/containerd/config.go @@ -58,6 +58,8 @@ func (c *config) decodePlugin(name string, v interface{}) error { type grpcConfig struct { Socket string `toml:"socket"` + Uid int `toml:"uid"` + Gid int `toml:"gid"` } type debug struct { diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go index b7224bc82..8e2a2346d 100644 --- a/cmd/containerd/main.go +++ b/cmd/containerd/main.go @@ -330,6 +330,9 @@ func serveGRPC(server *grpc.Server) error { if err != nil { return err } + if err := os.Chown(path, conf.GRPC.Uid, conf.GRPC.Gid); err != nil { + return err + } go func() { defer l.Close() if err := server.Serve(l); err != nil { From 4fd24785130963560819d9cb4d892bbf9d49c772 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 13:47:49 -0800 Subject: [PATCH 3/8] Update go-runc to bd9aef7cf4402a3a8728e3ef83dcca6a5a1be899 Also remove the comment fields in the vndr script, they are too hard to keep up to date and add little value when we have the git commit timestamps Signed-off-by: Michael Crosby --- vendor.conf | 34 +------- .../github.com/crosbymichael/go-runc/runc.go | 80 ++++++++++--------- .../github.com/crosbymichael/go-runc/utils.go | 17 ---- 3 files changed, 45 insertions(+), 86 deletions(-) diff --git a/vendor.conf b/vendor.conf index a20102a21..1054185b8 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,64 +1,32 @@ -# go-runc client for runc; master as of 01/20/2017 -github.com/crosbymichael/go-runc 706de6f422f397fb70b8c98f9b8c8eab2de32ae2 -# console pkg; +github.com/crosbymichael/go-runc bd9aef7cf4402a3a8728e3ef83dcca6a5a1be899 github.com/crosbymichael/console 4bf9d88357031b516b3794a2594b6d060a29c59c -# go-metrics client to prometheus; master as of 12/16/2016 github.com/docker/go-metrics 0f35294225552d968a13f9c5bc71a3fa44b2eb87 -# prometheus client; latest release as of 12/16/2016 github.com/prometheus/client_golang v0.8.0 -# prometheus client model; master as of 12/16/2016 github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 -# prometheus common library; master as of 12/16/2016 github.com/prometheus/common 195bde7883f7c39ea62b0d92ab7359b5327065cb -# prometheus procfs; master as of 12/16/2016 github.com/prometheus/procfs fcdb11ccb4389efb1b210b7ffb623ab71c5fdd60 -# beorn7/perks; master as of 12/16/2016 github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 -# matttproud/golang_protobuf_extensions; latest tagged release as of 12/16/2016 github.com/matttproud/golang_protobuf_extensions v1.0.0 -# go-units from Docker; latest release as of 12/16/2016 github.com/docker/go-units v0.3.1 -# gogo/protobuf - master as of 2/15/2016 (latest tagged release doesn't have needed change) github.com/gogo/protobuf d2e1ade2d719b78fe5b061b4c18a9f7111b5bdc8 -# golang support for protobufs - master as of 12/16/2016 github.com/golang/protobuf 8ee79997227bf9b34611aee7946ae64735e6fd93 -# runc, latest release as of 12/16/2016 github.com/opencontainers/runc ce450bcc6c135cae93ee2a99d41a308c179ff6dc -# OCI runtime spec, latest release as of 12/16/2016 github.com/opencontainers/runtime-spec v1.0.0-rc3 -# logrus, latest release as of 12/16/2016 github.com/Sirupsen/logrus v0.11.0 -# go-btrfs from stevvooe; master as of 1/11/2017 github.com/stevvooe/go-btrfs 8539a1d04898663b8eda14982e24b74e7a12388e -# testify go testing support; latest release as of 12/16/2016 github.com/stretchr/testify v1.1.4 -# go-spew (required by testify, and also by ctr); latest release as of 1/12/2017 github.com/davecgh/go-spew v1.1.0 -# go-difflib (required by testify); latest release as of 1/12/2017 github.com/pmezard/go-difflib v1.0.0 -# Go pkg for handling fifos; master as of 12/16/2016 github.com/tonistiigi/fifo fe870ccf293940774c2b44e23f6c71fff8f7547d -# client application library; latest release as of 12/16/2016 github.com/urfave/cli v1.19.1 -# extended Golang net package; upstream reported githash as of 12/16/2016 golang.org/x/net 8b4af36cd21a1f85a7484b49feb7c79363106d8e -# Go gRPC support; latest release as of 12/16/2016 google.golang.org/grpc v1.0.5 -# pkg/errors; latest release as of 12/16/2016 github.com/pkg/errors v0.8.0 -# lockfile; master as of 1/12/2017 github.com/nightlyone/lockfile 1d49c987357a327b5b03aa84cbddd582c328615d -# docker (for docker/pkg/archive, which is required by snapshot); latest experimental release as of 1/12/2017 github.com/docker/docker v1.13.0-rc6 -# go-digest; master as of 1/12/2017 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 -# sys/unix; master as of 1/12/2017 golang.org/x/sys/unix d75a52659825e75fff6158388dddc6a5b04f9ba5 -# image-spec master as of 1/17/2017 github.com/opencontainers/image-spec a431dbcf6a74fca2e0e040b819a836dbe3fb23ca -# continuity master as of 2/1/2017 github.com/stevvooe/continuity 1530f13d23b34e2ccaf33881fefecc7e28e3577b -# sync master as of 12/5/2016 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c - github.com/BurntSushi/toml v0.2.0-21-g9906417 diff --git a/vendor/github.com/crosbymichael/go-runc/runc.go b/vendor/github.com/crosbymichael/go-runc/runc.go index b228153c1..7b4cc6574 100644 --- a/vendor/github.com/crosbymichael/go-runc/runc.go +++ b/vendor/github.com/crosbymichael/go-runc/runc.go @@ -40,7 +40,7 @@ type Runc struct { // List returns all containers created inside the provided runc root directory func (r *Runc) List(context context.Context) ([]*Container, error) { - data, err := r.command(context, "list", "--format=json").Output() + data, err := Monitor.Output(r.command(context, "list", "--format=json")) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func (r *Runc) List(context context.Context) ([]*Container, error) { // State returns the state for the container provided by id func (r *Runc) State(context context.Context, id string) (*Container, error) { - data, err := r.command(context, "state", id).CombinedOutput() + data, err := Monitor.CombinedOutput(r.command(context, "state", id)) if err != nil { return nil, fmt.Errorf("%s: %s", err, data) } @@ -72,6 +72,7 @@ type CreateOpts struct { Detach bool NoPivot bool NoNewKeyring bool + ExtraFiles []*os.File } func (o *CreateOpts) args() (out []string, err error) { @@ -94,6 +95,9 @@ func (o *CreateOpts) args() (out []string, err error) { if o.Detach { out = append(out, "--detach") } + if o.ExtraFiles != nil { + out = append(out, "--preserve-fds", strconv.Itoa(len(o.ExtraFiles))) + } return out, nil } @@ -111,14 +115,16 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp if opts != nil && opts.IO != nil { opts.Set(cmd) } + cmd.ExtraFiles = opts.ExtraFiles + if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmd.CombinedOutput() + data, err := Monitor.CombinedOutput(cmd) if err != nil { return fmt.Errorf("%s: %s", err, data) } return nil } - if err := cmd.Start(); err != nil { + if err := Monitor.Start(cmd); err != nil { return err } if opts != nil && opts.IO != nil { @@ -128,36 +134,26 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp } } } - return cmd.Wait() + _, err := Monitor.Wait(cmd) + return err } // Start will start an already created container func (r *Runc) Start(context context.Context, id string) error { - return runOrError(r.command(context, "start", id)) + return r.runOrError(r.command(context, "start", id)) } type ExecOpts struct { IO PidFile string - Uid int - Gid int - Cwd string - Tty bool ConsoleSocket *ConsoleSocket Detach bool } func (o *ExecOpts) args() (out []string, err error) { - out = append(out, "--user", fmt.Sprintf("%d:%d", o.Uid, o.Gid)) - if o.Tty { - out = append(out, "--tty") - } if o.ConsoleSocket != nil { out = append(out, "--console-socket", o.ConsoleSocket.Path()) } - if o.Cwd != "" { - out = append(out, "--cwd", o.Cwd) - } if o.Detach { out = append(out, "--detach") } @@ -174,7 +170,7 @@ func (o *ExecOpts) args() (out []string, err error) { // Exec executres and additional process inside the container based on a full // OCI Process specification func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error { - f, err := ioutil.TempFile("", "-process") + f, err := ioutil.TempFile("", "runc-process") if err != nil { return err } @@ -197,13 +193,13 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts opts.Set(cmd) } if cmd.Stdout == nil && cmd.Stderr == nil { - data, err := cmd.CombinedOutput() + data, err := Monitor.CombinedOutput(cmd) if err != nil { return fmt.Errorf("%s: %s", err, data) } return nil } - if err := cmd.Start(); err != nil { + if err := Monitor.Start(cmd); err != nil { return err } if opts != nil && opts.IO != nil { @@ -213,7 +209,8 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts } } } - return cmd.Wait() + _, err = Monitor.Wait(cmd) + return err } // Run runs the create, start, delete lifecycle of the container @@ -231,19 +228,15 @@ func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) if opts != nil { opts.Set(cmd) } - if err := cmd.Start(); err != nil { + if err := Monitor.Start(cmd); err != nil { return -1, err } - status, err := cmd.Process.Wait() - if err != nil { - return -1, err - } - return status.Sys().(syscall.WaitStatus).ExitStatus(), nil + return Monitor.Wait(cmd) } // Delete deletes the container func (r *Runc) Delete(context context.Context, id string) error { - return runOrError(r.command(context, "delete", id)) + return r.runOrError(r.command(context, "delete", id)) } // KillOpts specifies options for killing a container and its processes @@ -266,7 +259,7 @@ func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) if opts != nil { args = append(args, opts.args()...) } - return runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...)) + return r.runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...)) } // Stats return the stats for a container like cpu, memory, and io @@ -278,9 +271,9 @@ func (r *Runc) Stats(context context.Context, id string) (*Stats, error) { } defer func() { rd.Close() - cmd.Wait() + Monitor.Wait(cmd) }() - if err := cmd.Start(); err != nil { + if err := Monitor.Start(cmd); err != nil { return nil, err } var e Event @@ -297,7 +290,7 @@ func (r *Runc) Events(context context.Context, id string, interval time.Duration if err != nil { return nil, err } - if err := cmd.Start(); err != nil { + if err := Monitor.Start(cmd); err != nil { rd.Close() return nil, err } @@ -309,7 +302,7 @@ func (r *Runc) Events(context context.Context, id string, interval time.Duration defer func() { close(c) rd.Close() - cmd.Wait() + Monitor.Wait(cmd) }() for { var e Event @@ -330,17 +323,17 @@ func (r *Runc) Events(context context.Context, id string, interval time.Duration // Pause the container with the provided id func (r *Runc) Pause(context context.Context, id string) error { - return runOrError(r.command(context, "pause", id)) + return r.runOrError(r.command(context, "pause", id)) } // Resume the container with the provided id func (r *Runc) Resume(context context.Context, id string) error { - return runOrError(r.command(context, "resume", id)) + return r.runOrError(r.command(context, "resume", id)) } // Ps lists all the processes inside the container returning their pids func (r *Runc) Ps(context context.Context, id string) ([]int, error) { - data, err := r.command(context, "ps", "--format", "json", id).CombinedOutput() + data, err := Monitor.CombinedOutput(r.command(context, "ps", "--format", "json", id)) if err != nil { return nil, fmt.Errorf("%s: %s", err, data) } @@ -380,3 +373,18 @@ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd { } return cmd } + +// runOrError will run the provided command. If an error is +// encountered and neither Stdout or Stderr was set the error and the +// stderr of the command will be returned in the format of : +// +func (r *Runc) runOrError(cmd *exec.Cmd) error { + if cmd.Stdout != nil || cmd.Stderr != nil { + return Monitor.Run(cmd) + } + data, err := Monitor.CombinedOutput(cmd) + if err != nil { + return fmt.Errorf("%s: %s", err, data) + } + return nil +} diff --git a/vendor/github.com/crosbymichael/go-runc/utils.go b/vendor/github.com/crosbymichael/go-runc/utils.go index 5ff327436..81fcd3f2d 100644 --- a/vendor/github.com/crosbymichael/go-runc/utils.go +++ b/vendor/github.com/crosbymichael/go-runc/utils.go @@ -1,9 +1,7 @@ package runc import ( - "fmt" "io/ioutil" - "os/exec" "strconv" "syscall" ) @@ -18,21 +16,6 @@ func ReadPidFile(path string) (int, error) { return strconv.Atoi(string(data)) } -// runOrError will run the provided command. If an error is -// encountered and neither Stdout or Stderr was set the error and the -// stderr of the command will be returned in the format of : -// -func runOrError(cmd *exec.Cmd) error { - if cmd.Stdout != nil || cmd.Stderr != nil { - return cmd.Run() - } - data, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%s: %s", err, data) - } - return nil -} - const exitSignalOffset = 128 // exitStatus returns the correct exit status for a process based on if it From df48983fe76262e68240004384fcbc86257a82af Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 16:07:35 -0800 Subject: [PATCH 4/8] Add reaper code for daemon Signed-off-by: Michael Crosby --- cmd/containerd/main.go | 7 ++- linux/runtime.go | 2 +- linux/shim.go | 5 +-- reaper/reaper.go | 100 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 reaper/reaper.go diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go index 8e2a2346d..69d44d0d5 100644 --- a/cmd/containerd/main.go +++ b/cmd/containerd/main.go @@ -21,6 +21,7 @@ import ( "github.com/docker/containerd/content" "github.com/docker/containerd/log" "github.com/docker/containerd/plugin" + "github.com/docker/containerd/reaper" "github.com/docker/containerd/snapshot" "github.com/docker/containerd/utils" metrics "github.com/docker/go-metrics" @@ -83,7 +84,7 @@ func main() { // start the signal handler as soon as we can to make sure that // we don't miss any signals during boot signals := make(chan os.Signal, 2048) - signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1) + signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGCHLD) log.G(global).Info("starting containerd boot...") // load all plugins into containerd @@ -363,6 +364,10 @@ func handleSignals(signals chan os.Signal, server *grpc.Server) error { for s := range signals { log.G(global).WithField("signal", s).Debug("received signal") switch s { + case syscall.SIGCHLD: + if err := reaper.Reap(); err != nil { + log.G(global).WithError(err).Error("reap containerd processes") + } default: server.Stop() return nil diff --git a/linux/runtime.go b/linux/runtime.go index 9901e8503..00a97d110 100644 --- a/linux/runtime.go +++ b/linux/runtime.go @@ -38,7 +38,7 @@ type Config struct { Runtime string `toml:"runtime"` } -func New(ic *containerd.InitContext) (interface{}, error) { +func New(ic *plugin.InitContext) (interface{}, error) { path := filepath.Join(ic.State, runtimeName) if err := os.MkdirAll(path, 0700); err != nil { return nil, err diff --git a/linux/shim.go b/linux/shim.go index 62cb03224..a01179641 100644 --- a/linux/shim.go +++ b/linux/shim.go @@ -14,6 +14,7 @@ import ( "google.golang.org/grpc/grpclog" "github.com/docker/containerd/api/services/shim" + "github.com/docker/containerd/reaper" "github.com/docker/containerd/utils" "github.com/pkg/errors" ) @@ -41,11 +42,9 @@ func newShim(path string) (shim.ShimClient, error) { Cloneflags: syscall.CLONE_NEWNS, Setpgid: true, } - if err := cmd.Start(); err != nil { + if err := reaper.Default.Start(cmd); err != nil { return nil, errors.Wrapf(err, "failed to start shim") } - // since we are currently the parent go ahead and make sure we wait on the shim - go cmd.Wait() return connectShim(socket) } diff --git a/reaper/reaper.go b/reaper/reaper.go new file mode 100644 index 000000000..dc4394d73 --- /dev/null +++ b/reaper/reaper.go @@ -0,0 +1,100 @@ +package reaper + +import ( + "bytes" + "fmt" + "os/exec" + "sync" + + "github.com/docker/containerd/utils" +) + +// Reap should be called when the process receives an SIGCHLD. Reap will reap +// all exited processes and close their wait channels +func Reap() error { + exits, err := utils.Reap(false) + for _, e := range exits { + Default.mu.Lock() + c, ok := Default.cmds[e.Pid] + Default.mu.Unlock() + if !ok { + continue + } + // after we get an exit, call wait on the go process to make sure all + // pipes are closed and finalizers are run on the process + c.c.Wait() + c.exitCh <- e.Status + Default.mu.Lock() + delete(Default.cmds, e.Pid) + Default.mu.Unlock() + } + return err +} + +var Default = &Monitor{ + cmds: make(map[int]*cmd), +} + +type Monitor struct { + mu sync.Mutex + cmds map[int]*cmd +} + +func (m *Monitor) Output(c *exec.Cmd) ([]byte, error) { + var b bytes.Buffer + c.Stdout = &b + if err := m.Run(c); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func (m *Monitor) CombinedOutput(c *exec.Cmd) ([]byte, error) { + var b bytes.Buffer + c.Stdout = &b + c.Stderr = &b + if err := m.Run(c); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// Start starts the command a registers the process with the reaper +func (m *Monitor) Start(c *exec.Cmd) error { + rc := &cmd{ + c: c, + exitCh: make(chan int, 1), + } + // make sure we register the command first before starting the process + m.mu.Lock() + // start the process + if err := rc.c.Start(); err != nil { + m.mu.Unlock() + return err + } + m.cmds[rc.c.Process.Pid] = rc + m.mu.Unlock() + return nil +} + +// Run runs and waits for the command to finish +func (m *Monitor) Run(c *exec.Cmd) error { + if err := m.Start(c); err != nil { + return err + } + _, err := m.Wait(c) + return err +} + +func (m *Monitor) Wait(c *exec.Cmd) (int, error) { + rc, ok := m.cmds[c.Process.Pid] + if !ok { + return 255, fmt.Errorf("process does not exist") + } + return <-rc.exitCh, nil +} + +type cmd struct { + c *exec.Cmd + exitCh chan int +} From 9f3240364f6d0a15c6d44ab159ac6ab3baa1fae7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 16:11:57 -0800 Subject: [PATCH 5/8] Implement reaper with runc support in shim Signed-off-by: Michael Crosby --- cmd/containerd-shim/main.go | 8 ++++++-- cmd/containerd/main.go | 2 +- reaper/reaper.go | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/containerd-shim/main.go b/cmd/containerd-shim/main.go index a9b229685..8a45ad830 100644 --- a/cmd/containerd-shim/main.go +++ b/cmd/containerd-shim/main.go @@ -11,11 +11,12 @@ import ( "google.golang.org/grpc" "github.com/Sirupsen/logrus" + runc "github.com/crosbymichael/go-runc" "github.com/docker/containerd" shimapi "github.com/docker/containerd/api/services/shim" "github.com/docker/containerd/linux/shim" + "github.com/docker/containerd/reaper" "github.com/docker/containerd/sys" - "github.com/docker/containerd/utils" "github.com/urfave/cli" ) @@ -78,6 +79,9 @@ func main() { func setupSignals() (chan os.Signal, error) { signals := make(chan os.Signal, 2048) signal.Notify(signals) + // make sure runc is setup to use the monitor + // for waiting on processes + runc.Monitor = reaper.Default // set the shim as the subreaper for all orphaned processes created by the container if err := sys.SetSubreaper(1); err != nil { return nil, err @@ -108,7 +112,7 @@ func handleSignals(signals chan os.Signal, server *grpc.Server, service *shim.Se logrus.WithField("signal", s).Debug("received signal") switch s { case syscall.SIGCHLD: - exits, err := utils.Reap(false) + exits, err := reaper.Reap() if err != nil { logrus.WithError(err).Error("reap exit status") } diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go index 69d44d0d5..ebce5a443 100644 --- a/cmd/containerd/main.go +++ b/cmd/containerd/main.go @@ -365,7 +365,7 @@ func handleSignals(signals chan os.Signal, server *grpc.Server) error { log.G(global).WithField("signal", s).Debug("received signal") switch s { case syscall.SIGCHLD: - if err := reaper.Reap(); err != nil { + if _, err := reaper.Reap(); err != nil { log.G(global).WithError(err).Error("reap containerd processes") } default: diff --git a/reaper/reaper.go b/reaper/reaper.go index dc4394d73..40ddd8e09 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -11,7 +11,7 @@ import ( // Reap should be called when the process receives an SIGCHLD. Reap will reap // all exited processes and close their wait channels -func Reap() error { +func Reap() ([]utils.Exit, error) { exits, err := utils.Reap(false) for _, e := range exits { Default.mu.Lock() @@ -28,7 +28,7 @@ func Reap() error { delete(Default.cmds, e.Pid) Default.mu.Unlock() } - return err + return exits, err } var Default = &Monitor{ From 6d3a70eb580ed83f8cb59ebac332dc56a99fa413 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 16:13:23 -0800 Subject: [PATCH 6/8] Make ctr --rootfs an absolute path Signed-off-by: Michael Crosby --- cmd/ctr/run.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/ctr/run.go b/cmd/ctr/run.go index 726d658b5..661cea375 100644 --- a/cmd/ctr/run.go +++ b/cmd/ctr/run.go @@ -193,11 +193,15 @@ var runCommand = cli.Command{ if err != nil { return err } + abs, err := filepath.Abs(context.String("rootfs")) + if err != nil { + return err + } // for ctr right now just do a bind mount rootfs := []*mount.Mount{ { Type: "bind", - Source: context.String("rootfs"), + Source: abs, Options: []string{ "rw", "rbind", From 61263bd77e7f89e5d62acc1c97652c85c308d433 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 16:26:14 -0800 Subject: [PATCH 7/8] Set containerd as subreaper Signed-off-by: Michael Crosby --- cmd/containerd/config.go | 2 ++ cmd/containerd/main.go | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/cmd/containerd/config.go b/cmd/containerd/config.go index 996481321..65e401831 100644 --- a/cmd/containerd/config.go +++ b/cmd/containerd/config.go @@ -44,6 +44,8 @@ type config struct { Snapshotter string `toml:"snapshotter"` // Plugins provides plugin specific configuration for the initialization of a plugin Plugins map[string]toml.Primitive `toml:"plugins"` + // Enable containerd as a subreaper + Subreaper bool `toml:"subreaper"` md toml.MetaData } diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go index ebce5a443..2c5b3e985 100644 --- a/cmd/containerd/main.go +++ b/cmd/containerd/main.go @@ -23,6 +23,7 @@ import ( "github.com/docker/containerd/plugin" "github.com/docker/containerd/reaper" "github.com/docker/containerd/snapshot" + "github.com/docker/containerd/sys" "github.com/docker/containerd/utils" metrics "github.com/docker/go-metrics" "github.com/pkg/errors" @@ -85,6 +86,12 @@ func main() { // we don't miss any signals during boot signals := make(chan os.Signal, 2048) signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1, syscall.SIGCHLD) + if conf.Subreaper { + log.G(global).Info("setting subreaper...") + if err := sys.SetSubreaper(1); err != nil { + return err + } + } log.G(global).Info("starting containerd boot...") // load all plugins into containerd From fe539560443220cf79c734f89291f37f07afb5d3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 9 Mar 2017 16:38:27 -0800 Subject: [PATCH 8/8] Add missing monitor file Signed-off-by: Michael Crosby --- reaper/reaper.go | 3 +- .../crosbymichael/go-runc/monitor.go | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/crosbymichael/go-runc/monitor.go diff --git a/reaper/reaper.go b/reaper/reaper.go index 40ddd8e09..38894fe98 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -65,7 +65,6 @@ func (m *Monitor) Start(c *exec.Cmd) error { c: c, exitCh: make(chan int, 1), } - // make sure we register the command first before starting the process m.mu.Lock() // start the process if err := rc.c.Start(); err != nil { @@ -87,7 +86,9 @@ func (m *Monitor) Run(c *exec.Cmd) error { } func (m *Monitor) Wait(c *exec.Cmd) (int, error) { + m.mu.Lock() rc, ok := m.cmds[c.Process.Pid] + m.mu.Unlock() if !ok { return 255, fmt.Errorf("process does not exist") } diff --git a/vendor/github.com/crosbymichael/go-runc/monitor.go b/vendor/github.com/crosbymichael/go-runc/monitor.go new file mode 100644 index 000000000..83eebb002 --- /dev/null +++ b/vendor/github.com/crosbymichael/go-runc/monitor.go @@ -0,0 +1,50 @@ +package runc + +import ( + "os/exec" + "syscall" +) + +var Monitor ProcessMonitor = &defaultMonitor{} + +// ProcessMonitor is an interface for process monitoring +// +// It allows daemons using go-runc to have a SIGCHLD handler +// to handle exits without introducing races between the handler +// and go's exec.Cmd +// These methods should match the methods exposed by exec.Cmd to provide +// a consistent experience for the caller +type ProcessMonitor interface { + Output(*exec.Cmd) ([]byte, error) + CombinedOutput(*exec.Cmd) ([]byte, error) + Run(*exec.Cmd) error + Start(*exec.Cmd) error + Wait(*exec.Cmd) (int, error) +} + +type defaultMonitor struct { +} + +func (m *defaultMonitor) Output(c *exec.Cmd) ([]byte, error) { + return c.Output() +} + +func (m *defaultMonitor) CombinedOutput(c *exec.Cmd) ([]byte, error) { + return c.CombinedOutput() +} + +func (m *defaultMonitor) Run(c *exec.Cmd) error { + return c.Run() +} + +func (m *defaultMonitor) Start(c *exec.Cmd) error { + return c.Start() +} + +func (m *defaultMonitor) Wait(c *exec.Cmd) (int, error) { + status, err := c.Process.Wait() + if err != nil { + return -1, err + } + return status.Sys().(syscall.WaitStatus).ExitStatus(), nil +}