Merge pull request #9661 from dmcgowan/update-cri-container-events-multisub

Add support for multiple subscribers to CRI container events

pkg/cri/server/container_events.go

@@ -21,10 +21,10 @@ import (
 )
 
 func (c *criService) GetContainerEvents(r *runtime.GetEventsRequest, s runtime.RuntimeService_GetContainerEventsServer) error {
-	// TODO (https://github.com/containerd/containerd/issues/7318):
-	// replace with a real implementation that broadcasts containerEventsChan
-	// to all subscribers.
-	for event := range c.containerEventsChan {
+	eventC, closer := c.containerEventsQ.Subscribe()
+	defer closer.Close()
+
+	for event := range eventC {
 		if err := s.Send(&event); err != nil {
 			return err
 		}

pkg/cri/server/helpers.go

@@ -395,13 +395,7 @@ func (c *criService) generateAndSendContainerEvent(ctx context.Context, containe
 		ContainersStatuses: containerStatuses,
 	}
 
-	// TODO(ruiwen-zhao): write events to a cache, storage, or increase the size of the channel
-	select {
-	case c.containerEventsChan <- event:
-	default:
-		containerEventsDroppedCount.Inc()
-		log.G(ctx).Debugf("containerEventsChan is full, discarding event %+v", event)
-	}
+	c.containerEventsQ.Send(event)
 }
 
 func (c *criService) getPodSandboxStatus(ctx context.Context, podSandboxID string) (*runtime.PodSandboxStatus, error) {

pkg/cri/server/service.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/containerd/go-cni"
 	"github.com/containerd/log"
@@ -32,6 +33,7 @@ import (
 
 	containerd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/core/sandbox"
+	"github.com/containerd/containerd/v2/internal/eventq"
 	"github.com/containerd/containerd/v2/internal/registrar"
 	criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
 	"github.com/containerd/containerd/v2/pkg/cri/nri"
@@ -118,9 +120,9 @@ type criService struct {
 	// allCaps is the list of the capabilities.
 	// When nil, parsed from CapEff of /proc/self/status.
 	allCaps []string //nolint:nolintlint,unused // Ignore on non-Linux
-	// containerEventsChan is used to capture container events and send them
-	// to the caller of GetContainerEvents.
-	containerEventsChan chan runtime.ContainerEventResponse
+	// containerEventsQ is used to capture container events and send them
+	// to the callers of GetContainerEvents.
+	containerEventsQ eventq.EventQueue[runtime.ContainerEventResponse]
 	// nri is used to hook NRI into CRI request processing.
 	nri *nri.API
 	// sandboxService is the sandbox related service for CRI
@@ -165,8 +167,15 @@ func NewCRIService(config criconfig.Config, options *CRIServiceOptions) (CRIServ
 		sandboxService:     newCriSandboxService(&config, options.Client),
 	}
 
-	// TODO: figure out a proper channel size.
-	c.containerEventsChan = make(chan runtime.ContainerEventResponse, 1000)
+	// TODO: Make discard time configurable
+	c.containerEventsQ = eventq.New[runtime.ContainerEventResponse](5*time.Minute, func(event runtime.ContainerEventResponse) {
+		containerEventsDroppedCount.Inc()
+		log.L.WithFields(
+			log.Fields{
+				"container": event.ContainerId,
+				"type":      event.ContainerEventType,
+			}).Warn("container event discarded")
+	})
 
 	if err := c.initPlatform(); err != nil {
 		return nil, nil, fmt.Errorf("initialize platform: %w", err)