From 2fecf5b02e95cdf4ee5babd065b2c52d6aadbe18 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 9 Sep 2021 23:08:39 +0000 Subject: [PATCH] Make sure exit signals trigger an exit during init Some cases can cause the server initialization to block (namely running a 2nd containerd instance by accident against the same root dir). In this case there is no way to quit the daemon except with `kill -9`. This changes context things so that server init is done in a goroutine and we wait on a channel for it to be ready while we also wait for a ctx.Done(), which will be cancelled if there is a termination signal. Signed-off-by: Brian Goff --- cmd/containerd/command/main.go | 68 ++++++++++++++++++++------ cmd/containerd/command/main_unix.go | 9 ++-- cmd/containerd/command/main_windows.go | 10 ++-- 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/cmd/containerd/command/main.go b/cmd/containerd/command/main.go index e3b9dbc05..ff633dff5 100644 --- a/cmd/containerd/command/main.go +++ b/cmd/containerd/command/main.go @@ -111,13 +111,15 @@ can be used and modified as necessary as a custom configuration.` } app.Action = func(context *cli.Context) error { var ( - start = time.Now() - signals = make(chan os.Signal, 2048) - serverC = make(chan *server.Server, 1) - ctx = gocontext.Background() - config = defaultConfig() + start = time.Now() + signals = make(chan os.Signal, 2048) + serverC = make(chan *server.Server, 1) + ctx, cancel = gocontext.WithCancel(gocontext.Background()) + config = defaultConfig() ) + defer cancel() + // Only try to load the config if it either exists, or the user explicitly // told us to load this path. configPath := context.GlobalString("config") @@ -161,7 +163,7 @@ can be used and modified as necessary as a custom configuration.` return nil } - done := handleSignals(ctx, signals, serverC) + done := handleSignals(ctx, signals, serverC, cancel) // start the signal handler as soon as we can to make sure that // we don't miss any signals during boot signal.Notify(signals, handledSignals...) @@ -193,18 +195,56 @@ can be used and modified as necessary as a custom configuration.` "revision": version.Revision, }).Info("starting containerd") - server, err := server.New(ctx, config) - if err != nil { - return err + type srvResp struct { + s *server.Server + err error + } + // run server initialization in a goroutine so we don't end up blocking important things like SIGTERM handling + // while the server is initializing. + // As an example opening the bolt database will block forever if another containerd is already running and containerd + // will have to be be `kill -9`'ed to recover. + chsrv := make(chan srvResp) + go func() { + defer close(chsrv) + + server, err := server.New(ctx, config) + if err != nil { + select { + case chsrv <- srvResp{err: err}: + case <-ctx.Done(): + } + return + } + + // Launch as a Windows Service if necessary + if err := launchService(server, done); err != nil { + logrus.Fatal(err) + } + select { + case <-ctx.Done(): + server.Stop() + case chsrv <- srvResp{s: server}: + } + }() + + var server *server.Server + select { + case <-ctx.Done(): + return ctx.Err() + case r := <-chsrv: + if r.err != nil { + return err + } + server = r.s } - // Launch as a Windows Service if necessary - if err := launchService(server, done); err != nil { - logrus.Fatal(err) + // We don't send the server down serverC directly in the goroutine above because we need it lower down. + select { + case <-ctx.Done(): + return ctx.Err() + case serverC <- server: } - serverC <- server - if config.Debug.Address != "" { var l net.Listener if isLocalAddress(config.Debug.Address) { diff --git a/cmd/containerd/command/main_unix.go b/cmd/containerd/command/main_unix.go index 9b2441fa1..e81de6387 100644 --- a/cmd/containerd/command/main_unix.go +++ b/cmd/containerd/command/main_unix.go @@ -36,7 +36,7 @@ var handledSignals = []os.Signal{ unix.SIGPIPE, } -func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} { +func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} { done := make(chan struct{}, 1) go func() { var server *server.Server @@ -61,11 +61,10 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se log.G(ctx).WithError(err).Error("notify stopping failed") } - if server == nil { - close(done) - return + cancel() + if server != nil { + server.Stop() } - server.Stop() close(done) return } diff --git a/cmd/containerd/command/main_windows.go b/cmd/containerd/command/main_windows.go index 1803e1808..602703527 100644 --- a/cmd/containerd/command/main_windows.go +++ b/cmd/containerd/command/main_windows.go @@ -39,7 +39,7 @@ var ( } ) -func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} { +func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} { done := make(chan struct{}) go func() { var server *server.Server @@ -54,12 +54,12 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se log.G(ctx).WithError(err).Error("notify stopping failed") } - if server == nil { - close(done) - return + cancel() + if server != nil { + server.Stop() } - server.Stop() close(done) + return } } }()