Let cri-containerd exit with containerd

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/server/service.go

@@ -18,14 +18,17 @@ package server
 
 import (
 	"fmt"
+	"net"
+	"os"
+	"syscall"
 
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/api/services/events/v1"
 	"github.com/containerd/containerd/api/services/tasks/v1"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	"github.com/cri-o/ocicni"
 	"github.com/golang/glog"
+	"google.golang.org/grpc"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 
@@ -36,18 +39,27 @@ import (
 	sandboxstore "github.com/kubernetes-incubator/cri-containerd/pkg/store/sandbox"
 )
 
-// k8sContainerdNamespace is the namespace we use to connect containerd.
-const k8sContainerdNamespace = "k8s.io"
+const (
+	// k8sContainerdNamespace is the namespace we use to connect containerd.
+	k8sContainerdNamespace = "k8s.io"
+	// unixProtocol is the network protocol of unix socket.
+	unixProtocol = "unix"
+)
 
 // CRIContainerdService is the interface implement CRI remote service server.
 type CRIContainerdService interface {
-	Start()
+	Run() error
+	Stop()
 	runtime.RuntimeServiceServer
 	runtime.ImageServiceServer
 }
 
 // criContainerdService implements CRIContainerdService.
 type criContainerdService struct {
+	// serverAddress is the grpc server unix path.
+	serverAddress string
+	// server is the grpc server.
+	server *grpc.Server
 	// os is an interface for all required os operations.
 	os osinterface.OS
 	// rootDir is the directory for managing cri-containerd files.
@@ -76,8 +88,6 @@ type criContainerdService struct {
 	// imageStoreService is the containerd service to store and track
 	// image metadata.
 	imageStoreService images.Store
-	// eventsService is the containerd task service client
-	eventService events.EventsClient
 	// netPlugin is used to setup and teardown network when run/stop pod sandbox.
 	netPlugin ocicni.CNIPlugin
 	// client is an instance of the containerd client
@@ -86,11 +96,14 @@ type criContainerdService struct {
 	streamServer streaming.Server
 	// cgroupPath in which the cri-containerd is placed in
 	cgroupPath string
+	// eventMonitor is the monitor monitors containerd events.
+	eventMonitor *eventMonitor
 }
 
 // NewCRIContainerdService returns a new instance of CRIContainerdService
 // TODO(random-liu): Add cri-containerd server config to get rid of the long arg list.
 func NewCRIContainerdService(
+	serverAddress,
 	containerdEndpoint,
 	containerdSnapshotter,
 	rootDir,
@@ -113,6 +126,7 @@ func NewCRIContainerdService(
 	}
 
 	c := &criContainerdService{
+		serverAddress:       serverAddress,
 		os:                  osinterface.RealOS{},
 		rootDir:             rootDir,
 		sandboxImage:        defaultSandboxImage,
@@ -124,7 +138,6 @@ func NewCRIContainerdService(
 		containerNameIndex:  registrar.NewRegistrar(),
 		taskService:         client.TaskService(),
 		imageStoreService:   client.ImageService(),
-		eventService:        client.EventService(),
 		contentStoreService: client.ContentStore(),
 		client:              client,
 		cgroupPath:          cgroupPath,
@@ -142,14 +155,78 @@ func NewCRIContainerdService(
 		return nil, fmt.Errorf("failed to create stream server: %v", err)
 	}
 
+	c.eventMonitor, err = newEventMonitor(c)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create event monitor: %v", err)
+	}
+
+	// Create the grpc server and register runtime and image services.
+	c.server = grpc.NewServer()
+	runtime.RegisterRuntimeServiceServer(c.server, newInstrumentedService(c))
+	runtime.RegisterImageServiceServer(c.server, newInstrumentedService(c))
+
 	return newInstrumentedService(c), nil
 }
 
-func (c *criContainerdService) Start() {
-	c.startEventMonitor()
+// Run starts the cri-containerd service.
+func (c *criContainerdService) Run() error {
+	glog.V(2).Info("Start cri-containerd service")
+	// TODO(random-liu): Recover state.
+
+	// Start event handler.
+	glog.V(2).Info("Start event monitor")
+	eventMonitorCloseCh := c.eventMonitor.start()
+
+	// Start streaming server.
+	glog.V(2).Info("Start streaming server")
+	streamServerCloseCh := make(chan struct{})
 	go func() {
 		if err := c.streamServer.Start(true); err != nil {
 			glog.Errorf("Failed to start streaming server: %v", err)
 		}
+		close(streamServerCloseCh)
 	}()
+
+	// Start grpc server.
+	// Unlink to cleanup the previous socket file.
+	glog.V(2).Info("Start grpc server")
+	err := syscall.Unlink(c.serverAddress)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to unlink socket file %q: %v", c.serverAddress, err)
+	}
+	l, err := net.Listen(unixProtocol, c.serverAddress)
+	if err != nil {
+		return fmt.Errorf("failed to listen on %q: %v", c.serverAddress, err)
+	}
+	grpcServerCloseCh := make(chan struct{})
+	go func() {
+		if err := c.server.Serve(l); err != nil {
+			glog.Errorf("Failed to serve grpc grpc request: %v", err)
+		}
+		close(grpcServerCloseCh)
+	}()
+
+	// Stop the whole cri-containerd service if any of the critical service exits.
+	select {
+	case <-eventMonitorCloseCh:
+	case <-streamServerCloseCh:
+	case <-grpcServerCloseCh:
+	}
+	c.Stop()
+
+	<-eventMonitorCloseCh
+	glog.V(2).Info("Event monitor stopped")
+	<-streamServerCloseCh
+	glog.V(2).Info("Stream server stopped")
+	<-grpcServerCloseCh
+	glog.V(2).Info("GRPC server stopped")
+	return nil
+}
+
+// Stop stops the cri-containerd service.
+func (c *criContainerdService) Stop() {
+	glog.V(2).Info("Stop cri-containerd service")
+	c.eventMonitor.stop()
+	c.streamServer.Stop() // nolint: errcheck
+	c.server.Stop()
 }