github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #126429 from saschagrunert/kubelet-panic

Browse Source 
Fix kubelet cadvisor stats runtime panic
This commit is contained in:
Kubernetes Prow Robot
2024-07-29 11:06:07 -07:00
committed by GitHub
parent aab56e9b70 50e430b3e9
commit e8588e6493
2 changed files with 100 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@@ -270,26 +270,34 @@ func (p *cadvisorStatsProvider) ImageFsStats(ctx context.Context) (imageFsRet *s
return imageFs, imageFs, nil
}
imageStats, err := p.imageService.ImageFsInfo(ctx)
if err != nil || imageStats == nil {
return nil, nil, fmt.Errorf("failed to get image stats: %v", err)
if err != nil {
return nil, nil, fmt.Errorf("failed to get image stats: %w", err)
}
if imageStats == nil || len(imageStats.ImageFilesystems) == 0 || len(imageStats.ContainerFilesystems) == 0 {
return nil, nil, fmt.Errorf("missing image stats: %+v", imageStats)
}
splitFileSystem := false
if imageStats.ImageFilesystems[0].FsId.Mountpoint != imageStats.ContainerFilesystems[0].FsId.Mountpoint {
klog.InfoS("Detect Split Filesystem", "ImageFilesystems", imageStats.ImageFilesystems[0], "ContainerFilesystems", imageStats.ContainerFilesystems[0])
imageFs := imageStats.ImageFilesystems[0]
containerFs := imageStats.ContainerFilesystems[0]
if imageFs.FsId != nil && containerFs.FsId != nil && imageFs.FsId.Mountpoint != containerFs.FsId.Mountpoint {
klog.InfoS("Detect Split Filesystem", "ImageFilesystems", imageFs, "ContainerFilesystems", containerFs)
splitFileSystem = true
}
imageFs := imageStats.ImageFilesystems[0]
var imageFsInodesUsed *uint64
if imageFsInfo.Inodes != nil && imageFsInfo.InodesFree != nil {
imageFsIU := *imageFsInfo.Inodes - *imageFsInfo.InodesFree
imageFsInodesUsed = &imageFsIU
}
var usedBytes uint64
if imageFs.GetUsedBytes() != nil {
usedBytes = imageFs.GetUsedBytes().GetValue()
}
fsStats := &statsapi.FsStats{
Time: metav1.NewTime(imageFsInfo.Timestamp),
AvailableBytes: &imageFsInfo.Available,
CapacityBytes: &imageFsInfo.Capacity,
UsedBytes: &imageFs.UsedBytes.Value,
UsedBytes: &usedBytes,
InodesFree: imageFsInfo.InodesFree,
Inodes: imageFsInfo.Inodes,
InodesUsed: imageFsInodesUsed,
@@ -305,18 +313,21 @@ func (p *cadvisorStatsProvider) ImageFsStats(ctx context.Context) (imageFsRet *s
return nil, nil, fmt.Errorf("failed to get container fs info: %w", err)
}
containerFs := imageStats.ContainerFilesystems[0]
var containerFsInodesUsed *uint64
if containerFsInfo.Inodes != nil && containerFsInfo.InodesFree != nil {
containerFsIU := *containerFsInfo.Inodes - *containerFsInfo.InodesFree
containerFsInodesUsed = &containerFsIU
}
var usedContainerBytes uint64
if containerFs.GetUsedBytes() != nil {
usedContainerBytes = containerFs.GetUsedBytes().GetValue()
}
fsContainerStats := &statsapi.FsStats{
Time: metav1.NewTime(containerFsInfo.Timestamp),
AvailableBytes: &containerFsInfo.Available,
CapacityBytes: &containerFsInfo.Capacity,
UsedBytes: &containerFs.UsedBytes.Value,
UsedBytes: &usedContainerBytes,
InodesFree: containerFsInfo.InodesFree,
Inodes: containerFsInfo.Inodes,
InodesUsed: containerFsInodesUsed,

81
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View File

@@ -23,6 +23,7 @@ import (
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -546,6 +547,86 @@ func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) {
assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
}
func TestImageFsStatsCustomResponse(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletSeparateDiskGC, true)
for desc, tc := range map[string]struct {
response *runtimeapi.ImageFsInfoResponse
callContainerFsInfo, shouldErr bool
}{
"image stats are nil": {
shouldErr: true,
},
"no image filesystems in image stats": {
response: &runtimeapi.ImageFsInfoResponse{
ImageFilesystems: []*runtimeapi.FilesystemUsage{},
ContainerFilesystems: []*runtimeapi.FilesystemUsage{{}},
},
shouldErr: true,
},
"no container filesystems in image stats": {
response: &runtimeapi.ImageFsInfoResponse{
ImageFilesystems: []*runtimeapi.FilesystemUsage{{}},
ContainerFilesystems: []*runtimeapi.FilesystemUsage{},
},
shouldErr: true,
},
"image and container filesystem identifiers are nil": {
response: &runtimeapi.ImageFsInfoResponse{
ImageFilesystems: []*runtimeapi.FilesystemUsage{{}},
ContainerFilesystems: []*runtimeapi.FilesystemUsage{{}},
},
shouldErr: false,
},
"using different mountpoints but no used bytes set": {
response: &runtimeapi.ImageFsInfoResponse{
ImageFilesystems: []*runtimeapi.FilesystemUsage{{
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-1"},
}},
ContainerFilesystems: []*runtimeapi.FilesystemUsage{{
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-2"},
}},
},
callContainerFsInfo: true,
shouldErr: false,
},
"using different mountpoints and set used bytes": {
response: &runtimeapi.ImageFsInfoResponse{
ImageFilesystems: []*runtimeapi.FilesystemUsage{{
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-1"},
UsedBytes: &runtimeapi.UInt64Value{Value: 10},
}},
ContainerFilesystems: []*runtimeapi.FilesystemUsage{{
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: "mnt-2"},
UsedBytes: &runtimeapi.UInt64Value{Value: 20},
}},
},
callContainerFsInfo: true,
shouldErr: false,
},
} {
ctx := context.Background()
mockCadvisor := cadvisortest.NewMockInterface(t)
mockRuntime := containertest.NewMockRuntime(t)
res := getTestFsInfo(1000)
mockCadvisor.EXPECT().ImagesFsInfo().Return(res, nil)
mockRuntime.EXPECT().ImageFsInfo(ctx).Return(tc.response, nil)
if tc.callContainerFsInfo {
mockCadvisor.EXPECT().ContainerFsInfo().Return(res, nil)
}
provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider())
stats, containerfs, err := provider.ImageFsStats(ctx)
if tc.shouldErr {
require.Error(t, err, desc)
assert.Nil(t, stats)
assert.Nil(t, containerfs)
} else {
assert.NoError(t, err, desc)
}
}
}
func TestCadvisorImagesFsStats(t *testing.T) {
ctx := context.Background()
var (