Make sure exit signals trigger an exit during init

Some cases can cause the server initialization to block (namely running
a 2nd containerd instance by accident against the same root dir). In
this case there is no way to quit the daemon except with `kill -9`.

This changes context things so that server init is done in a goroutine
and we wait on a channel for it to be ready while we also wait for a
ctx.Done(), which will be cancelled if there is a termination signal.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

cmd/containerd/command/main.go

@@ -114,10 +114,12 @@ can be used and modified as necessary as a custom configuration.`
 			start       = time.Now()
 			signals     = make(chan os.Signal, 2048)
 			serverC     = make(chan *server.Server, 1)
-			ctx     = gocontext.Background()
+			ctx, cancel = gocontext.WithCancel(gocontext.Background())
 			config      = defaultConfig()
 		)
 
+		defer cancel()
+
 		// Only try to load the config if it either exists, or the user explicitly
 		// told us to load this path.
 		configPath := context.GlobalString("config")
@@ -161,7 +163,7 @@ can be used and modified as necessary as a custom configuration.`
 			return nil
 		}
 
-		done := handleSignals(ctx, signals, serverC)
+		done := handleSignals(ctx, signals, serverC, cancel)
 		// start the signal handler as soon as we can to make sure that
 		// we don't miss any signals during boot
 		signal.Notify(signals, handledSignals...)
@@ -193,17 +195,55 @@ can be used and modified as necessary as a custom configuration.`
 			"revision": version.Revision,
 		}).Info("starting containerd")
 
+		type srvResp struct {
+			s   *server.Server
+			err error
+		}
+		// run server initialization in a goroutine so we don't end up blocking important things like SIGTERM handling
+		// while the server is initializing.
+		// As an example opening the bolt database will block forever if another containerd is already running and containerd
+		// will have to be be `kill -9`'ed to recover.
+		chsrv := make(chan srvResp)
+		go func() {
+			defer close(chsrv)
+
 			server, err := server.New(ctx, config)
 			if err != nil {
-			return err
+				select {
+				case chsrv <- srvResp{err: err}:
+				case <-ctx.Done():
+				}
+				return
 			}
 
 			// Launch as a Windows Service if necessary
 			if err := launchService(server, done); err != nil {
 				logrus.Fatal(err)
 			}
+			select {
+			case <-ctx.Done():
+				server.Stop()
+			case chsrv <- srvResp{s: server}:
+			}
+		}()
 
-		serverC <- server
+		var server *server.Server
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case r := <-chsrv:
+			if r.err != nil {
+				return err
+			}
+			server = r.s
+		}
+
+		// We don't send the server down serverC directly in the goroutine above because we need it lower down.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case serverC <- server:
+		}
 
 		if config.Debug.Address != "" {
 			var l net.Listener

cmd/containerd/command/main_unix.go

@@ -36,7 +36,7 @@ var handledSignals = []os.Signal{
 	unix.SIGPIPE,
 }
 
-func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} {
+func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} {
 	done := make(chan struct{}, 1)
 	go func() {
 		var server *server.Server
@@ -61,11 +61,10 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se
 						log.G(ctx).WithError(err).Error("notify stopping failed")
 					}
 
-					if server == nil {
-						close(done)
-						return
-					}
+					cancel()
+					if server != nil {
 						server.Stop()
+					}
 					close(done)
 					return
 				}

cmd/containerd/command/main_windows.go

@@ -39,7 +39,7 @@ var (
 	}
 )
 
-func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server) chan struct{} {
+func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *server.Server, cancel func()) chan struct{} {
 	done := make(chan struct{})
 	go func() {
 		var server *server.Server
@@ -54,13 +54,13 @@ func handleSignals(ctx context.Context, signals chan os.Signal, serverC chan *se
 					log.G(ctx).WithError(err).Error("notify stopping failed")
 				}
 
-				if server == nil {
+				cancel()
+				if server != nil {
+					server.Stop()
+				}
 				close(done)
 				return
 			}
-				server.Stop()
-				close(done)
-			}
 		}
 	}()
 	setupDumpStacks()