github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

CRI: enhance ImageFsInfo() to support multiple snapshotters

Browse Source 
Enhance cri/server/image/imagefs_info.go:ImageFsInfo() to support
snapshotter per runtime. Now `ImageFsInfoResponse.ImageFilesystems` may
contain multiple entries.

Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>
This commit is contained in:
Jiang Liu 2023-10-16 10:21:44 +08:00
parent 5ad6f34329
commit 8e7c10c6d0
7 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/cri/server/container_stats_list.go
View File

@ -291,7 +291,7 @@ func (c *criService) windowsContainerMetrics(
cs.WritableLayer = &runtime.FilesystemUsage{
Timestamp: sn.Timestamp,
FsId: &runtime.FilesystemIdentifier{
Mountpoint: c.imageFSPath,
Mountpoint: c.imageFSPaths[snapshotter],
},
UsedBytes: &runtime.UInt64Value{Value: usedBytes},
InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
@ -347,7 +347,7 @@ func (c *criService) linuxContainerMetrics(
cs.WritableLayer = &runtime.FilesystemUsage{
Timestamp: sn.Timestamp,
FsId: &runtime.FilesystemIdentifier{
Mountpoint: c.imageFSPath,
Mountpoint: c.imageFSPaths[snapshotter],
},
UsedBytes: &runtime.UInt64Value{Value: usedBytes},
InodesUsed: &runtime.UInt64Value{Value: inodesUsed},

68
pkg/cri/server/images/imagefs_info.go
View File

@ -20,32 +20,64 @@ import (
"context"
"time"
"github.com/containerd/containerd/pkg/cri/store/snapshot"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)
// ImageFsInfo returns information of the filesystem that is used to store images.
// TODO(windows): Usage for windows is always 0 right now. Support this for windows.
// TODO(random-liu): Handle storage consumed by content store
func (c *CRIImageService) ImageFsInfo(ctx context.Context, r *runtime.ImageFsInfoRequest) (*runtime.ImageFsInfoResponse, error) {
snapshots := c.snapshotStore.List()
timestamp := time.Now().UnixNano()
var usedBytes, inodesUsed uint64
snapshotterFSInfos := map[string]snapshot.Snapshot{}
for _, sn := range snapshots {
// Use the oldest timestamp as the timestamp of imagefs info.
if sn.Timestamp < timestamp {
timestamp = sn.Timestamp
if info, ok := snapshotterFSInfos[sn.Key.Snapshotter]; ok {
// Use the oldest timestamp as the timestamp of imagefs info.
if sn.Timestamp < info.Timestamp {
info.Timestamp = sn.Timestamp
}
info.Size += sn.Size
info.Inodes += sn.Inodes
snapshotterFSInfos[sn.Key.Snapshotter] = info
} else {
snapshotterFSInfos[sn.Key.Snapshotter] = snapshot.Snapshot{
Timestamp: sn.Timestamp,
Size: sn.Size,
Inodes: sn.Inodes,
}
}
usedBytes += sn.Size
inodesUsed += sn.Inodes
}
// TODO(random-liu): Handle content store
return &runtime.ImageFsInfoResponse{
ImageFilesystems: []*runtime.FilesystemUsage{
{
Timestamp: timestamp,
FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPath},
UsedBytes: &runtime.UInt64Value{Value: usedBytes},
InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
},
},
}, nil
var imageFilesystems []*runtime.FilesystemUsage
// Currently kubelet always consumes the first entry of the returned array,
// so put the default snapshotter as the first entry for compatibility.
if info, ok := snapshotterFSInfos[c.config.Snapshotter]; ok {
imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
Timestamp: info.Timestamp,
FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]},
UsedBytes: &runtime.UInt64Value{Value: info.Size},
InodesUsed: &runtime.UInt64Value{Value: info.Inodes},
})
delete(snapshotterFSInfos, c.config.Snapshotter)
} else {
imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
Timestamp: time.Now().UnixNano(),
FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[c.config.Snapshotter]},
UsedBytes: &runtime.UInt64Value{Value: 0},
InodesUsed: &runtime.UInt64Value{Value: 0},
})
}
for snapshotter, info := range snapshotterFSInfos {
imageFilesystems = append(imageFilesystems, &runtime.FilesystemUsage{
Timestamp: info.Timestamp,
FsId: &runtime.FilesystemIdentifier{Mountpoint: c.imageFSPaths[snapshotter]},
UsedBytes: &runtime.UInt64Value{Value: info.Size},
InodesUsed: &runtime.UInt64Value{Value: info.Inodes},
})
}
return &runtime.ImageFsInfoResponse{ImageFilesystems: imageFilesystems}, nil
}

11
pkg/cri/server/images/imagefs_info_test.go
View File

@ -34,7 +34,7 @@ func TestImageFsInfo(t *testing.T) {
{
Key: snapshotstore.Key{
Key: "key1",
Snapshotter: "snapshotter1",
Snapshotter: "overlayfs",
},
Kind: snapshot.KindActive,
Size: 10,
@ -44,7 +44,7 @@ func TestImageFsInfo(t *testing.T) {
{
Key: snapshotstore.Key{
Key: "key2",
Snapshotter: "snapshotter1",
Snapshotter: "overlayfs",
},
Kind: snapshot.KindCommitted,
Size: 20,
@ -54,7 +54,7 @@ func TestImageFsInfo(t *testing.T) {
{
Key: snapshotstore.Key{
Key: "key3",
Snapshotter: "snapshotter1",
Snapshotter: "overlayfs",
},
Kind: snapshot.KindView,
Size: 0,
@ -74,6 +74,7 @@ func TestImageFsInfo(t *testing.T) {
resp, err := c.ImageFsInfo(context.Background(), &runtime.ImageFsInfoRequest{})
require.NoError(t, err)
stats := resp.GetImageFilesystems()
assert.Len(t, stats, 1)
assert.Equal(t, expected, stats[0])
// stats[0] is for default snapshotter, stats[1] is for `overlayfs`
assert.Len(t, stats, 2)
assert.Equal(t, expected, stats[1])
}

8
pkg/cri/server/images/service.go
View File

@ -39,8 +39,8 @@ type CRIImageService struct {
config criconfig.Config
// client is an instance of the containerd client
client *containerd.Client
// imageFSPath is the path to image filesystem.
imageFSPath string
// imageFSPaths contains path to image filesystem for snapshotters.
imageFSPaths map[string]string
// imageStore stores all resources associated with images.
imageStore *imagestore.Store
// snapshotStore stores information of all snapshots.
@ -51,12 +51,12 @@ type CRIImageService struct {
unpackDuplicationSuppressor kmutex.KeyedLocker
}
func NewService(config criconfig.Config, imageFSPath string, client *containerd.Client) (*CRIImageService, error) {
func NewService(config criconfig.Config, imageFSPaths map[string]string, client *containerd.Client) (*CRIImageService, error) {
svc := CRIImageService{
config: config,
client: client,
imageStore: imagestore.NewStore(client.ImageService(), client.ContentStore(), platforms.Default()),
imageFSPath: imageFSPath,
imageFSPaths: imageFSPaths,
snapshotStore: snapshotstore.NewStore(),
unpackDuplicationSuppressor: kmutex.New(),
}

2
pkg/cri/server/images/service_test.go
View File

@ -42,7 +42,7 @@ const (
func newTestCRIService() *CRIImageService {
return &CRIImageService{
config: testConfig,
imageFSPath: testImageFSPath,
imageFSPaths: map[string]string{"overlayfs": testImageFSPath},
imageStore: imagestore.NewStore(nil, nil, platforms.Default()),
snapshotStore: snapshotstore.NewStore(),
}

2
pkg/cri/server/sandbox_stats_windows.go
View File

@ -173,7 +173,7 @@ func (c *criService) toPodSandboxStats(sandbox sandboxstore.Sandbox, statsMap ma
containerStats.WritableLayer = &runtime.WindowsFilesystemUsage{
Timestamp: sn.Timestamp,
FsId: &runtime.FilesystemIdentifier{
Mountpoint: c.imageFSPath,
Mountpoint: c.imageFSPaths[snapshotter],
},
UsedBytes: &runtime.UInt64Value{Value: usedBytes},
}

22
pkg/cri/server/service.go
View File

@ -87,8 +87,8 @@ type criService struct {
imageService
// config contains all configurations.
config criconfig.Config
// imageFSPath is the path to image filesystem.
imageFSPath string
// imageFSPaths contains path to image filesystem for snapshotters.
imageFSPaths map[string]string
// os is an interface for all required os operations.
os osinterface.OS
// sandboxStore stores all resources associated with sandboxes.
@ -139,11 +139,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter)
}
imageFSPath := imageFSPath(config.ContainerdRootDir, config.ContainerdConfig.Snapshotter)
log.L.Infof("Get image filesystem path %q", imageFSPath)
imageFSPaths := map[string]string{}
for _, ociRuntime := range config.ContainerdConfig.Runtimes {
// Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.`
snapshotter := ociRuntime.Snapshotter
if snapshotter != "" {
imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
}
}
snapshotter := config.ContainerdConfig.Snapshotter
imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter)
log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter)
// TODO: expose this as a separate containerd plugin.
imageService, err := images.NewService(config, imageFSPath, client)
imageService, err := images.NewService(config, imageFSPaths, client)
if err != nil {
return nil, fmt.Errorf("unable to create CRI image service: %w", err)
}
@ -152,7 +162,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.
imageService: imageService,
config: config,
client: client,
imageFSPath: imageFSPath,
imageFSPaths: imageFSPaths,
os: osinterface.RealOS{},
sandboxStore: sandboxstore.NewStore(labels),
containerStore: containerstore.NewStore(labels),