CRI: enhance ImageFsInfo() to support multiple snapshotters

Enhance cri/server/image/imagefs_info.go:ImageFsInfo() to support
snapshotter per runtime. Now `ImageFsInfoResponse.ImageFilesystems` may
contain multiple entries.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>

pkg/cri/server/container_stats_list.go

@@ -291,7 +291,7 @@ func (c *criService) windowsContainerMetrics(
 	cs.WritableLayer = &runtime.FilesystemUsage{
 		Timestamp: sn.Timestamp,
 		FsId: &runtime.FilesystemIdentifier{
-			Mountpoint: c.imageFSPath,
+			Mountpoint: c.imageFSPaths[snapshotter],
 		},
 		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
 		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
@@ -347,7 +347,7 @@ func (c *criService) linuxContainerMetrics(
 	cs.WritableLayer = &runtime.FilesystemUsage{
 		Timestamp: sn.Timestamp,
 		FsId: &runtime.FilesystemIdentifier{
-			Mountpoint: c.imageFSPath,
+			Mountpoint: c.imageFSPaths[snapshotter],
 		},
 		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
 		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},

pkg/cri/server/images/imagefs_info.go

@@ -20,32 +20,64 @@ import (
 	"context"
 	"time"
 
+	"github.com/containerd/containerd/pkg/cri/store/snapshot"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
 // TODO(windows): Usage for windows is always 0 right now. Support this for windows.
+// TODO(random-liu): Handle storage consumed by content store
 func (c *CRIImageService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (*runtime.ImageFsInfoResponse, error) {
 	snapshots := c.snapshotStore.List()
-	timestamp := time.Now().UnixNano()
+	snapshotterFSInfos := map[string]snapshot.Snapshot{}
-	var usedBytes, inodesUsed uint64
+
 	for _, sn := range snapshots {
+		if info, ok := snapshotterFSInfos[sn.Key.Snapshotter]; ok {
 			// Use the oldest timestamp as the timestamp of imagefs info.
-		if sn.Timestamp < timestamp {
+			if sn.Timestamp < info.Timestamp {
-			timestamp = sn.Timestamp
+				info.Timestamp = sn.Timestamp
 			}
-		usedBytes += sn.Size
+			info.Size += sn.Size
-		inodesUsed += sn.Inodes
+			info.Inodes += sn.Inodes
+			snapshotterFSInfos[sn.Key.Snapshotter] = info
+		} else {
+			snapshotterFSInfos[sn.Key.Snapshotter] = snapshot.Snapshot{
+				Timestamp: sn.Timestamp,
+				Size:      sn.Size,
+				Inodes:    sn.Inodes,
 			}
-	// TODO(random-liu): Handle content store
+		}
-	return &runtime.ImageFsInfoResponse{
+	}
-		ImageFilesystems: []*runtime.FilesystemUsage{
+
-			{
+	var imageFilesystems []*runtime.FilesystemUsage
-				Timestamp:  timestamp,
+
-				FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPath},
+	// Currently kubelet always consumes the first entry of the returned array,
-				UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+	// so put the default snapshotter as the first entry for compatibility.
-				InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	if info, ok := snapshotterFSInfos[c.config.Snapshotter]; ok {
-			},
+		imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
-		},
+			Timestamp:  info.Timestamp,
-	}, nil
+			FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]},
+			UsedBytes:  &runtime.UInt64Value{Value: info.Size},
+			InodesUsed: &runtime.UInt64Value{Value: info.Inodes},
+		})
+		delete(snapshotterFSInfos, c.config.Snapshotter)
+	} else {
+		imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
+			Timestamp:  time.Now().UnixNano(),
+			FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]},
+			UsedBytes:  &runtime.UInt64Value{Value: 0},
+			InodesUsed: &runtime.UInt64Value{Value: 0},
+		})
+	}
+
+	for snapshotter, info := range snapshotterFSInfos {
+		imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
+			Timestamp:  info.Timestamp,
+			FsId:       &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[snapshotter]},
+			UsedBytes:  &runtime.UInt64Value{Value: info.Size},
+			InodesUsed: &runtime.UInt64Value{Value: info.Inodes},
+		})
+	}
+
+	return &runtime.ImageFsInfoResponse{ImageFilesystems: imageFilesystems}, nil
 }

pkg/cri/server/images/imagefs_info_test.go

@@ -34,7 +34,7 @@ func TestImageFsInfo(t *testing.T) {
 		{
 			Key: snapshotstore.Key{
 				Key:         "key1",
-				Snapshotter: "snapshotter1",
+				Snapshotter: "overlayfs",
 			},
 			Kind:      snapshot.KindActive,
 			Size:      10,
@@ -44,7 +44,7 @@ func TestImageFsInfo(t *testing.T) {
 		{
 			Key: snapshotstore.Key{
 				Key:         "key2",
-				Snapshotter: "snapshotter1",
+				Snapshotter: "overlayfs",
 			},
 			Kind:      snapshot.KindCommitted,
 			Size:      20,
@@ -54,7 +54,7 @@ func TestImageFsInfo(t *testing.T) {
 		{
 			Key: snapshotstore.Key{
 				Key:         "key3",
-				Snapshotter: "snapshotter1",
+				Snapshotter: "overlayfs",
 			},
 			Kind:      snapshot.KindView,
 			Size:      0,
@@ -74,6 +74,7 @@ func TestImageFsInfo(t *testing.T) {
 	resp, err := c.ImageFsInfo(context.Background(), &runtime.ImageFsInfoRequest{})
 	require.NoError(t, err)
 	stats := resp.GetImageFilesystems()
-	assert.Len(t, stats, 1)
+	// stats[0] is for default snapshotter, stats[1] is for `overlayfs`
-	assert.Equal(t, expected, stats[0])
+	assert.Len(t, stats, 2)
+	assert.Equal(t, expected, stats[1])
 }

pkg/cri/server/images/service.go

@@ -39,8 +39,8 @@ type CRIImageService struct {
 	config criconfig.Config
 	// client is an instance of the containerd client
 	client *containerd.Client
-	// imageFSPath is the path to image filesystem.
+	// imageFSPaths contains path to image filesystem for snapshotters.
-	imageFSPath string
+	imageFSPaths map[string]string
 	// imageStore stores all resources associated with images.
 	imageStore *imagestore.Store
 	// snapshotStore stores information of all snapshots.
@@ -51,12 +51,12 @@ type CRIImageService struct {
 	unpackDuplicationSuppressor kmutex.KeyedLocker
 }
 
-func NewService(config criconfig.Config, imageFSPath string, client *containerd.Client) (*CRIImageService, error) {
+func NewService(config criconfig.Config, imageFSPaths map[string]string, client *containerd.Client) (*CRIImageService, error) {
 	svc := CRIImageService{
 		config:                      config,
 		client:                      client,
 		imageStore:                  imagestore.NewStore(client.ImageService(), client.ContentStore(), platforms.Default()),
-		imageFSPath:                 imageFSPath,
+		imageFSPaths:                imageFSPaths,
 		snapshotStore:               snapshotstore.NewStore(),
 		unpackDuplicationSuppressor: kmutex.New(),
 	}

pkg/cri/server/images/service_test.go

@@ -42,7 +42,7 @@ const (
 func newTestCRIService() *CRIImageService {
 	return &CRIImageService{
 		config:        testConfig,
-		imageFSPath:   testImageFSPath,
+		imageFSPaths:  map[string]string{"overlayfs": testImageFSPath},
 		imageStore:    imagestore.NewStore(nil, nil, platforms.Default()),
 		snapshotStore: snapshotstore.NewStore(),
 	}

pkg/cri/server/sandbox_stats_windows.go

@@ -173,7 +173,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma
 		containerStats.WritableLayer = &runtime.WindowsFilesystemUsage{
 			Timestamp: sn.Timestamp,
 			FsId: &runtime.FilesystemIdentifier{
-				Mountpoint: c.imageFSPath,
+				Mountpoint: c.imageFSPaths[snapshotter],
 			},
 			UsedBytes: &runtime.UInt64Value{Value: usedBytes},
 		}

pkg/cri/server/service.go

@@ -87,8 +87,8 @@ type criService struct {
 	imageService
 	// config contains all configurations.
 	config criconfig.Config
-	// imageFSPath is the path to image filesystem.
+	// imageFSPaths contains path to image filesystem for snapshotters.
-	imageFSPath string
+	imageFSPaths map[string]string
 	// os is an interface for all required os operations.
 	os osinterface.OS
 	// sandboxStore stores all resources associated with sandboxes.
@@ -139,11 +139,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter)
 	}
 
-	imageFSPath := imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter)
+	imageFSPaths := map[string]string{}
-	log.L.Infof("Get image filesystem path %q", imageFSPath)
+	for _, ociRuntime := range config.ContainerdConfig.Runtimes {
+		// Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.`
+		snapshotter := ociRuntime.Snapshotter
+		if snapshotter != "" {
+			imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
+			log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
+		}
+	}
+	snapshotter := config.ContainerdConfig.Snapshotter
+	imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
+	log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
 
 	// TODO: expose this as a separate containerd plugin.
-	imageService, err := images.NewService(config, imageFSPath, client)
+	imageService, err := images.NewService(config, imageFSPaths, client)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create CRI image service: %w", err)
 	}
@@ -152,7 +162,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
 		imageService:       imageService,
 		config:             config,
 		client:             client,
-		imageFSPath:        imageFSPath,
+		imageFSPaths:       imageFSPaths,
 		os:                 osinterface.RealOS{},
 		sandboxStore:       sandboxstore.NewStore(labels),
 		containerStore:     containerstore.NewStore(labels),