From 8e7c10c6d088138192a98f59986cbbd026d32a22 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Mon, 16 Oct 2023 10:21:44 +0800 Subject: [PATCH] CRI: enhance ImageFsInfo() to support multiple snapshotters Enhance cri/server/image/imagefs_info.go:ImageFsInfo() to support snapshotter per runtime. Now `ImageFsInfoResponse.ImageFilesystems` may contain multiple entries. Signed-off-by: Jiang Liu --- pkg/cri/server/container_stats_list.go | 4 +- pkg/cri/server/images/imagefs_info.go | 68 ++++++++++++++++------ pkg/cri/server/images/imagefs_info_test.go | 11 ++-- pkg/cri/server/images/service.go | 8 +-- pkg/cri/server/images/service_test.go | 2 +- pkg/cri/server/sandbox_stats_windows.go | 2 +- pkg/cri/server/service.go | 22 +++++-- 7 files changed, 80 insertions(+), 37 deletions(-) diff --git a/pkg/cri/server/container_stats_list.go b/pkg/cri/server/container_stats_list.go index fc367c0af..3f3253c97 100644 --- a/pkg/cri/server/container_stats_list.go +++ b/pkg/cri/server/container_stats_list.go @@ -291,7 +291,7 @@ func (c *criService) windowsContainerMetrics( cs.WritableLayer = &runtime.FilesystemUsage{ Timestamp: sn.Timestamp, FsId: &runtime.FilesystemIdentifier{ - Mountpoint: c.imageFSPath, + Mountpoint: c.imageFSPaths[snapshotter], }, UsedBytes: &runtime.UInt64Value{Value: usedBytes}, InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, @@ -347,7 +347,7 @@ func (c *criService) linuxContainerMetrics( cs.WritableLayer = &runtime.FilesystemUsage{ Timestamp: sn.Timestamp, FsId: &runtime.FilesystemIdentifier{ - Mountpoint: c.imageFSPath, + Mountpoint: c.imageFSPaths[snapshotter], }, UsedBytes: &runtime.UInt64Value{Value: usedBytes}, InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, diff --git a/pkg/cri/server/images/imagefs_info.go b/pkg/cri/server/images/imagefs_info.go index f9e7ec631..0ee3331ea 100644 --- a/pkg/cri/server/images/imagefs_info.go +++ b/pkg/cri/server/images/imagefs_info.go @@ -20,32 +20,64 @@ import ( "context" "time" + "github.com/containerd/containerd/pkg/cri/store/snapshot" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) // ImageFsInfo returns information of the filesystem that is used to store images. // TODO(windows): Usage for windows is always 0 right now. Support this for windows. +// TODO(random-liu): Handle storage consumed by content store func (c *CRIImageService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (*runtime.ImageFsInfoResponse, error) { snapshots := c.snapshotStore.List() - timestamp := time.Now().UnixNano() - var usedBytes, inodesUsed uint64 + snapshotterFSInfos := map[string]snapshot.Snapshot{} + for _, sn := range snapshots { - // Use the oldest timestamp as the timestamp of imagefs info. - if sn.Timestamp < timestamp { - timestamp = sn.Timestamp + if info, ok := snapshotterFSInfos[sn.Key.Snapshotter]; ok { + // Use the oldest timestamp as the timestamp of imagefs info. + if sn.Timestamp < info.Timestamp { + info.Timestamp = sn.Timestamp + } + info.Size += sn.Size + info.Inodes += sn.Inodes + snapshotterFSInfos[sn.Key.Snapshotter] = info + } else { + snapshotterFSInfos[sn.Key.Snapshotter] = snapshot.Snapshot{ + Timestamp: sn.Timestamp, + Size: sn.Size, + Inodes: sn.Inodes, + } } - usedBytes += sn.Size - inodesUsed += sn.Inodes } - // TODO(random-liu): Handle content store - return &runtime.ImageFsInfoResponse{ - ImageFilesystems: []*runtime.FilesystemUsage{ - { - Timestamp: timestamp, - FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPath}, - UsedBytes: &runtime.UInt64Value{Value: usedBytes}, - InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, - }, - }, - }, nil + + var imageFilesystems []*runtime.FilesystemUsage + + // Currently kubelet always consumes the first entry of the returned array, + // so put the default snapshotter as the first entry for compatibility. + if info, ok := snapshotterFSInfos[c.config.Snapshotter]; ok { + imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{ + Timestamp: info.Timestamp, + FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]}, + UsedBytes: &runtime.UInt64Value{Value: info.Size}, + InodesUsed: &runtime.UInt64Value{Value: info.Inodes}, + }) + delete(snapshotterFSInfos, c.config.Snapshotter) + } else { + imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{ + Timestamp: time.Now().UnixNano(), + FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]}, + UsedBytes: &runtime.UInt64Value{Value: 0}, + InodesUsed: &runtime.UInt64Value{Value: 0}, + }) + } + + for snapshotter, info := range snapshotterFSInfos { + imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{ + Timestamp: info.Timestamp, + FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[snapshotter]}, + UsedBytes: &runtime.UInt64Value{Value: info.Size}, + InodesUsed: &runtime.UInt64Value{Value: info.Inodes}, + }) + } + + return &runtime.ImageFsInfoResponse{ImageFilesystems: imageFilesystems}, nil } diff --git a/pkg/cri/server/images/imagefs_info_test.go b/pkg/cri/server/images/imagefs_info_test.go index 647f2d83c..3d380999e 100644 --- a/pkg/cri/server/images/imagefs_info_test.go +++ b/pkg/cri/server/images/imagefs_info_test.go @@ -34,7 +34,7 @@ func TestImageFsInfo(t *testing.T) { { Key: snapshotstore.Key{ Key: "key1", - Snapshotter: "snapshotter1", + Snapshotter: "overlayfs", }, Kind: snapshot.KindActive, Size: 10, @@ -44,7 +44,7 @@ func TestImageFsInfo(t *testing.T) { { Key: snapshotstore.Key{ Key: "key2", - Snapshotter: "snapshotter1", + Snapshotter: "overlayfs", }, Kind: snapshot.KindCommitted, Size: 20, @@ -54,7 +54,7 @@ func TestImageFsInfo(t *testing.T) { { Key: snapshotstore.Key{ Key: "key3", - Snapshotter: "snapshotter1", + Snapshotter: "overlayfs", }, Kind: snapshot.KindView, Size: 0, @@ -74,6 +74,7 @@ func TestImageFsInfo(t *testing.T) { resp, err := c.ImageFsInfo(context.Background(), &runtime.ImageFsInfoRequest{}) require.NoError(t, err) stats := resp.GetImageFilesystems() - assert.Len(t, stats, 1) - assert.Equal(t, expected, stats[0]) + // stats[0] is for default snapshotter, stats[1] is for `overlayfs` + assert.Len(t, stats, 2) + assert.Equal(t, expected, stats[1]) } diff --git a/pkg/cri/server/images/service.go b/pkg/cri/server/images/service.go index 12b097aa2..e6b6a2a0f 100644 --- a/pkg/cri/server/images/service.go +++ b/pkg/cri/server/images/service.go @@ -39,8 +39,8 @@ type CRIImageService struct { config criconfig.Config // client is an instance of the containerd client client *containerd.Client - // imageFSPath is the path to image filesystem. - imageFSPath string + // imageFSPaths contains path to image filesystem for snapshotters. + imageFSPaths map[string]string // imageStore stores all resources associated with images. imageStore *imagestore.Store // snapshotStore stores information of all snapshots. @@ -51,12 +51,12 @@ type CRIImageService struct { unpackDuplicationSuppressor kmutex.KeyedLocker } -func NewService(config criconfig.Config, imageFSPath string, client *containerd.Client) (*CRIImageService, error) { +func NewService(config criconfig.Config, imageFSPaths map[string]string, client *containerd.Client) (*CRIImageService, error) { svc := CRIImageService{ config: config, client: client, imageStore: imagestore.NewStore(client.ImageService(), client.ContentStore(), platforms.Default()), - imageFSPath: imageFSPath, + imageFSPaths: imageFSPaths, snapshotStore: snapshotstore.NewStore(), unpackDuplicationSuppressor: kmutex.New(), } diff --git a/pkg/cri/server/images/service_test.go b/pkg/cri/server/images/service_test.go index bf480bbab..1ec7195de 100644 --- a/pkg/cri/server/images/service_test.go +++ b/pkg/cri/server/images/service_test.go @@ -42,7 +42,7 @@ const ( func newTestCRIService() *CRIImageService { return &CRIImageService{ config: testConfig, - imageFSPath: testImageFSPath, + imageFSPaths: map[string]string{"overlayfs": testImageFSPath}, imageStore: imagestore.NewStore(nil, nil, platforms.Default()), snapshotStore: snapshotstore.NewStore(), } diff --git a/pkg/cri/server/sandbox_stats_windows.go b/pkg/cri/server/sandbox_stats_windows.go index efb96e8c4..1691399b5 100644 --- a/pkg/cri/server/sandbox_stats_windows.go +++ b/pkg/cri/server/sandbox_stats_windows.go @@ -173,7 +173,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma containerStats.WritableLayer = &runtime.WindowsFilesystemUsage{ Timestamp: sn.Timestamp, FsId: &runtime.FilesystemIdentifier{ - Mountpoint: c.imageFSPath, + Mountpoint: c.imageFSPaths[snapshotter], }, UsedBytes: &runtime.UInt64Value{Value: usedBytes}, } diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index e8d0f4132..eb2dfe704 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -87,8 +87,8 @@ type criService struct { imageService // config contains all configurations. config criconfig.Config - // imageFSPath is the path to image filesystem. - imageFSPath string + // imageFSPaths contains path to image filesystem for snapshotters. + imageFSPaths map[string]string // os is an interface for all required os operations. os osinterface.OS // sandboxStore stores all resources associated with sandboxes. @@ -139,11 +139,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter) } - imageFSPath := imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter) - log.L.Infof("Get image filesystem path %q", imageFSPath) + imageFSPaths := map[string]string{} + for _, ociRuntime := range config.ContainerdConfig.Runtimes { + // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` + snapshotter := ociRuntime.Snapshotter + if snapshotter != "" { + imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) + log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) + } + } + snapshotter := config.ContainerdConfig.Snapshotter + imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) + log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) // TODO: expose this as a separate containerd plugin. - imageService, err := images.NewService(config, imageFSPath, client) + imageService, err := images.NewService(config, imageFSPaths, client) if err != nil { return nil, fmt.Errorf("unable to create CRI image service: %w", err) } @@ -152,7 +162,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri. imageService: imageService, config: config, client: client, - imageFSPath: imageFSPath, + imageFSPaths: imageFSPaths, os: osinterface.RealOS{}, sandboxStore: sandboxstore.NewStore(labels), containerStore: containerstore.NewStore(labels),