diff --git a/pkg/cri/sbserver/container_stats_list.go b/pkg/cri/sbserver/container_stats_list.go index 0da61d174..c1bbfedbe 100644 --- a/pkg/cri/sbserver/container_stats_list.go +++ b/pkg/cri/sbserver/container_stats_list.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/api/services/tasks/v1" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/cri/store/stats" "github.com/containerd/containerd/protobuf" "github.com/containerd/typeurl/v2" @@ -115,6 +116,14 @@ func (c *criService) toCRIContainerStats( if !ok { handler, err = c.getMetricsHandler(ctx, cntr.SandboxID) if err != nil { + // If the sandbox is not found, it may have been removed. we need to check container whether it is still exist + if errdefs.IsNotFound(err) { + _, err = c.containerStore.Get(cntr.ID) + if err != nil && errdefs.IsNotFound(err) { + log.G(ctx).Warnf("container %q is not found, skip it", cntr.ID) + continue + } + } return nil, fmt.Errorf("failed to get metrics handler for container %q: %w", cntr.ID, err) } sandboxToMetricsHandler[cntr.SandboxID] = handler diff --git a/pkg/cri/sbserver/container_stats_list_test.go b/pkg/cri/sbserver/container_stats_list_test.go index 8766504fc..0a7bf877d 100644 --- a/pkg/cri/sbserver/container_stats_list_test.go +++ b/pkg/cri/sbserver/container_stats_list_test.go @@ -17,13 +17,17 @@ package sbserver import ( + "context" "math" + "reflect" "testing" "time" v1 "github.com/containerd/cgroups/v3/cgroup1/stats" v2 "github.com/containerd/cgroups/v3/cgroup2/stats" + "github.com/containerd/containerd/api/types" containerstore "github.com/containerd/containerd/pkg/cri/store/container" + sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox" "github.com/stretchr/testify/assert" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) @@ -331,3 +335,103 @@ func TestContainerMetricsMemory(t *testing.T) { }) } } + +func TestListContainerStats(t *testing.T) { + c := newTestCRIService() + type args struct { + ctx context.Context + stats []*types.Metric + containers []containerstore.Container + } + tests := []struct { + name string + args args + before func() + after func() + want *runtime.ListContainerStatsResponse + wantErr bool + }{ + { + name: "args containers having c1,but containerStore not found c1, so filter c1", + args: args{ + ctx: context.Background(), + stats: []*types.Metric{ + { + ID: "c1", + }, + }, + containers: []containerstore.Container{ + { + Metadata: containerstore.Metadata{ + ID: "c1", + SandboxID: "s1", + }, + }, + }, + }, + want: &runtime.ListContainerStatsResponse{}, + }, + { + name: "args containers having c1,c2, but containerStore not found c1, so filter c1", + args: args{ + ctx: context.Background(), + stats: []*types.Metric{ + { + ID: "c1", + }, + { + ID: "c2", + }, + }, + containers: []containerstore.Container{ + { + Metadata: containerstore.Metadata{ + ID: "c1", + SandboxID: "s1", + }, + }, + { + Metadata: containerstore.Metadata{ + ID: "c2", + SandboxID: "s2", + }, + }, + }, + }, + before: func() { + c.containerStore.Add(containerstore.Container{ + Metadata: containerstore.Metadata{ + ID: "c2", + }, + }) + c.sandboxStore.Add(sandboxstore.Sandbox{ + Metadata: sandboxstore.Metadata{ + ID: "s2", + }, + }) + }, + wantErr: true, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.before != nil { + tt.before() + } + got, err := c.toCRIContainerStats(tt.args.ctx, tt.args.stats, tt.args.containers) + if tt.after != nil { + tt.after() + } + if (err != nil) != tt.wantErr { + t.Errorf("ListContainerStats() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ListContainerStats() = %v, want %v", got, tt.want) + } + }) + } + +}