diff --git a/pkg/cri/sbserver/container_stats.go b/pkg/cri/sbserver/container_stats.go index cf870c77c..6b643f88f 100644 --- a/pkg/cri/sbserver/container_stats.go +++ b/pkg/cri/sbserver/container_stats.go @@ -40,7 +40,12 @@ func (c *criService) ContainerStats(ctx context.Context, in *runtime.ContainerSt return nil, fmt.Errorf("unexpected metrics response: %+v", resp.Metrics) } - cs, err := c.containerMetrics(cntr.Metadata, resp.Metrics[0]) + handler, err := c.getMetricsHandler(ctx, cntr.SandboxID) + if err != nil { + return nil, err + } + + cs, err := handler(cntr.Metadata, resp.Metrics[0]) if err != nil { return nil, fmt.Errorf("failed to decode container metrics: %w", err) } diff --git a/pkg/cri/sbserver/container_stats_list.go b/pkg/cri/sbserver/container_stats_list.go index 41dee93ad..0da61d174 100644 --- a/pkg/cri/sbserver/container_stats_list.go +++ b/pkg/cri/sbserver/container_stats_list.go @@ -18,12 +18,20 @@ package sbserver import ( "context" + "errors" "fmt" + "reflect" "time" + wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" + cg1 "github.com/containerd/cgroups/v3/cgroup1/stats" + cg2 "github.com/containerd/cgroups/v3/cgroup2/stats" "github.com/containerd/containerd/api/services/tasks/v1" "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/pkg/cri/store/stats" + "github.com/containerd/containerd/protobuf" + "github.com/containerd/typeurl/v2" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" containerstore "github.com/containerd/containerd/pkg/cri/store/container" @@ -42,14 +50,48 @@ func (c *criService) ListContainerStats( if err != nil { return nil, fmt.Errorf("failed to fetch metrics for tasks: %w", err) } - criStats, err := c.toCRIContainerStats(resp.Metrics, containers) + criStats, err := c.toCRIContainerStats(ctx, resp.Metrics, containers) if err != nil { return nil, fmt.Errorf("failed to convert to cri containerd stats format: %w", err) } return criStats, nil } +type metricsHandler func(containerstore.Metadata, *types.Metric) (*runtime.ContainerStats, error) + +// Returns a function to be used for transforming container metrics into the right format. +// Uses the platform the given sandbox advertises to implement its logic. If the platform is +// unsupported for metrics this will return a wrapped [errdefs.ErrNotImplemented]. +func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (metricsHandler, error) { + sandbox, err := c.sandboxStore.Get(sandboxID) + if err != nil { + return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err) + } + controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler) + if err != nil { + return nil, fmt.Errorf("failed to get sandbox controller: %w", err) + } + // Grab the platform that this containers sandbox advertises. Reason being, even if + // the host may be {insert platform}, if it virtualizes or emulates a different platform + // it will return stats in that format, and we need to handle the conversion logic based + // off of this info. + p, err := controller.Platform(ctx, sandboxID) + if err != nil { + return nil, err + } + + switch p.OS { + case "windows": + return c.windowsContainerMetrics, nil + case "linux": + return c.linuxContainerMetrics, nil + default: + return nil, fmt.Errorf("container metrics for platform %+v: %w", p, errdefs.ErrNotImplemented) + } +} + func (c *criService) toCRIContainerStats( + ctx context.Context, stats []*types.Metric, containers []containerstore.Container, ) (*runtime.ListContainerStatsResponse, error) { @@ -58,8 +100,29 @@ func (c *criService) toCRIContainerStats( statsMap[stat.ID] = stat } containerStats := new(runtime.ListContainerStatsResponse) + + // Unfortunately if no filter was passed we're asking for every containers stats which + // generally belong to multiple different pods, who all might have different platforms. + // To avoid recalculating the right metricsHandler to invoke, if we've already calculated + // the platform and handler for a given sandbox just pull it from our map here. + var ( + err error + handler metricsHandler + ) + sandboxToMetricsHandler := make(map[string]metricsHandler) for _, cntr := range containers { - cs, err := c.containerMetrics(cntr.Metadata, statsMap[cntr.ID]) + h, ok := sandboxToMetricsHandler[cntr.SandboxID] + if !ok { + handler, err = c.getMetricsHandler(ctx, cntr.SandboxID) + if err != nil { + return nil, fmt.Errorf("failed to get metrics handler for container %q: %w", cntr.ID, err) + } + sandboxToMetricsHandler[cntr.SandboxID] = handler + } else { + handler = h + } + + cs, err := handler(cntr.Metadata, statsMap[cntr.ID]) if err != nil { return nil, fmt.Errorf("failed to decode container metrics for %q: %w", cntr.ID, err) } @@ -72,7 +135,6 @@ func (c *criService) toCRIContainerStats( } cs.Cpu.UsageNanoCores = &runtime.UInt64Value{Value: nanoUsage} } - containerStats.Stats = append(containerStats.Stats, cs) } return containerStats, nil @@ -133,7 +195,6 @@ func (c *criService) getUsageNanoCores(containerID string, isSandbox bool, curre if err != nil { return 0, fmt.Errorf("failed to update sandbox container stats: %s: %w", containerID, err) } - } else { err := c.containerStore.UpdateContainerStats(containerID, newStats) if err != nil { @@ -193,3 +254,238 @@ func matchLabelSelector(selector, labels map[string]string) bool { } return true } + +func (c *criService) windowsContainerMetrics( + meta containerstore.Metadata, + stats *types.Metric, +) (*runtime.ContainerStats, error) { + var cs runtime.ContainerStats + var usedBytes, inodesUsed uint64 + sn, err := c.GetSnapshot(meta.ID) + // If snapshotstore doesn't have cached snapshot information + // set WritableLayer usage to zero + if err == nil { + usedBytes = sn.Size + inodesUsed = sn.Inodes + } + cs.WritableLayer = &runtime.FilesystemUsage{ + Timestamp: sn.Timestamp, + FsId: &runtime.FilesystemIdentifier{ + Mountpoint: c.imageFSPath, + }, + UsedBytes: &runtime.UInt64Value{Value: usedBytes}, + InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, + } + cs.Attributes = &runtime.ContainerAttributes{ + Id: meta.ID, + Metadata: meta.Config.GetMetadata(), + Labels: meta.Config.GetLabels(), + Annotations: meta.Config.GetAnnotations(), + } + + if stats != nil { + s, err := typeurl.UnmarshalAny(stats.Data) + if err != nil { + return nil, fmt.Errorf("failed to extract container metrics: %w", err) + } + wstats := s.(*wstats.Statistics).GetWindows() + if wstats == nil { + return nil, errors.New("windows stats is empty") + } + if wstats.Processor != nil { + cs.Cpu = &runtime.CpuUsage{ + Timestamp: wstats.Timestamp.UnixNano(), + UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS}, + } + } + if wstats.Memory != nil { + cs.Memory = &runtime.MemoryUsage{ + Timestamp: wstats.Timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{ + Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes, + }, + } + } + } + return &cs, nil +} + +func (c *criService) linuxContainerMetrics( + meta containerstore.Metadata, + stats *types.Metric, +) (*runtime.ContainerStats, error) { + var cs runtime.ContainerStats + var usedBytes, inodesUsed uint64 + sn, err := c.GetSnapshot(meta.ID) + // If snapshotstore doesn't have cached snapshot information + // set WritableLayer usage to zero + if err == nil { + usedBytes = sn.Size + inodesUsed = sn.Inodes + } + cs.WritableLayer = &runtime.FilesystemUsage{ + Timestamp: sn.Timestamp, + FsId: &runtime.FilesystemIdentifier{ + Mountpoint: c.imageFSPath, + }, + UsedBytes: &runtime.UInt64Value{Value: usedBytes}, + InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, + } + cs.Attributes = &runtime.ContainerAttributes{ + Id: meta.ID, + Metadata: meta.Config.GetMetadata(), + Labels: meta.Config.GetLabels(), + Annotations: meta.Config.GetAnnotations(), + } + + if stats != nil { + var data interface{} + switch { + case typeurl.Is(stats.Data, (*cg1.Metrics)(nil)): + data = &cg1.Metrics{} + case typeurl.Is(stats.Data, (*cg2.Metrics)(nil)): + data = &cg2.Metrics{} + case typeurl.Is(stats.Data, (*wstats.Statistics)(nil)): + data = &wstats.Statistics{} + default: + return nil, errors.New("cannot convert metric data to cgroups.Metrics or windows.Statistics") + } + + if err := typeurl.UnmarshalTo(stats.Data, data); err != nil { + return nil, fmt.Errorf("failed to extract container metrics: %w", err) + } + + cpuStats, err := c.cpuContainerStats(meta.ID, false /* isSandbox */, data, protobuf.FromTimestamp(stats.Timestamp)) + if err != nil { + return nil, fmt.Errorf("failed to obtain cpu stats: %w", err) + } + cs.Cpu = cpuStats + + memoryStats, err := c.memoryContainerStats(meta.ID, data, protobuf.FromTimestamp(stats.Timestamp)) + if err != nil { + return nil, fmt.Errorf("failed to obtain memory stats: %w", err) + } + cs.Memory = memoryStats + } + + return &cs, nil +} + +// getWorkingSet calculates workingset memory from cgroup memory stats. +// The caller should make sure memory is not nil. +// workingset = usage - total_inactive_file +func getWorkingSet(memory *cg1.MemoryStat) uint64 { + if memory.Usage == nil { + return 0 + } + var workingSet uint64 + if memory.TotalInactiveFile < memory.Usage.Usage { + workingSet = memory.Usage.Usage - memory.TotalInactiveFile + } + return workingSet +} + +// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats. +// The caller should make sure memory is not nil. +// workingset = usage - inactive_file +func getWorkingSetV2(memory *cg2.MemoryStat) uint64 { + var workingSet uint64 + if memory.InactiveFile < memory.Usage { + workingSet = memory.Usage - memory.InactiveFile + } + return workingSet +} + +func isMemoryUnlimited(v uint64) bool { + // Size after which we consider memory to be "unlimited". This is not + // MaxInt64 due to rounding by the kernel. + // TODO: k8s or cadvisor should export this https://github.com/google/cadvisor/blob/2b6fbacac7598e0140b5bc8428e3bdd7d86cf5b9/metrics/prometheus.go#L1969-L1971 + const maxMemorySize = uint64(1 << 62) + + return v > maxMemorySize +} + +// https://github.com/kubernetes/kubernetes/blob/b47f8263e18c7b13dba33fba23187e5e0477cdbd/pkg/kubelet/stats/helper.go#L68-L71 +func getAvailableBytes(memory *cg1.MemoryStat, workingSetBytes uint64) uint64 { + // memory limit - working set bytes + if !isMemoryUnlimited(memory.Usage.Limit) { + return memory.Usage.Limit - workingSetBytes + } + return 0 +} + +func getAvailableBytesV2(memory *cg2.MemoryStat, workingSetBytes uint64) uint64 { + // memory limit (memory.max) for cgroupv2 - working set bytes + if !isMemoryUnlimited(memory.UsageLimit) { + return memory.UsageLimit - workingSetBytes + } + return 0 +} + +func (c *criService) cpuContainerStats(ID string, isSandbox bool, stats interface{}, timestamp time.Time) (*runtime.CpuUsage, error) { + switch metrics := stats.(type) { + case *cg1.Metrics: + metrics.GetCPU().GetUsage() + if metrics.CPU != nil && metrics.CPU.Usage != nil { + return &runtime.CpuUsage{ + Timestamp: timestamp.UnixNano(), + UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total}, + }, nil + } + case *cg2.Metrics: + if metrics.CPU != nil { + // convert to nano seconds + usageCoreNanoSeconds := metrics.CPU.UsageUsec * 1000 + + return &runtime.CpuUsage{ + Timestamp: timestamp.UnixNano(), + UsageCoreNanoSeconds: &runtime.UInt64Value{Value: usageCoreNanoSeconds}, + }, nil + } + default: + return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath()) + } + return nil, nil +} + +func (c *criService) memoryContainerStats(ID string, stats interface{}, timestamp time.Time) (*runtime.MemoryUsage, error) { + switch metrics := stats.(type) { + case *cg1.Metrics: + if metrics.Memory != nil && metrics.Memory.Usage != nil { + workingSetBytes := getWorkingSet(metrics.Memory) + + return &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{ + Value: workingSetBytes, + }, + AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytes(metrics.Memory, workingSetBytes)}, + UsageBytes: &runtime.UInt64Value{Value: metrics.Memory.Usage.Usage}, + RssBytes: &runtime.UInt64Value{Value: metrics.Memory.TotalRSS}, + PageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgFault}, + MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgMajFault}, + }, nil + } + case *cg2.Metrics: + if metrics.Memory != nil { + workingSetBytes := getWorkingSetV2(metrics.Memory) + + return &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{ + Value: workingSetBytes, + }, + AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytesV2(metrics.Memory, workingSetBytes)}, + UsageBytes: &runtime.UInt64Value{Value: metrics.Memory.Usage}, + // Use Anon memory for RSS as cAdvisor on cgroupv2 + // see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799 + RssBytes: &runtime.UInt64Value{Value: metrics.Memory.Anon}, + PageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgfault}, + MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgmajfault}, + }, nil + } + default: + return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath()) + } + return nil, nil +} diff --git a/pkg/cri/sbserver/container_stats_list_linux.go b/pkg/cri/sbserver/container_stats_list_linux.go deleted file mode 100644 index 231d51190..000000000 --- a/pkg/cri/sbserver/container_stats_list_linux.go +++ /dev/null @@ -1,213 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "errors" - "fmt" - "reflect" - "time" - - wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" - v1 "github.com/containerd/cgroups/v3/cgroup1/stats" - v2 "github.com/containerd/cgroups/v3/cgroup2/stats" - "github.com/containerd/containerd/api/types" - "github.com/containerd/containerd/protobuf" - "github.com/containerd/typeurl/v2" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - containerstore "github.com/containerd/containerd/pkg/cri/store/container" -) - -func (c *criService) containerMetrics( - meta containerstore.Metadata, - stats *types.Metric, -) (*runtime.ContainerStats, error) { - var cs runtime.ContainerStats - var usedBytes, inodesUsed uint64 - sn, err := c.GetSnapshot(meta.ID) - // If snapshotstore doesn't have cached snapshot information - // set WritableLayer usage to zero - if err == nil { - usedBytes = sn.Size - inodesUsed = sn.Inodes - } - cs.WritableLayer = &runtime.FilesystemUsage{ - Timestamp: sn.Timestamp, - FsId: &runtime.FilesystemIdentifier{ - Mountpoint: c.imageFSPath, - }, - UsedBytes: &runtime.UInt64Value{Value: usedBytes}, - InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, - } - cs.Attributes = &runtime.ContainerAttributes{ - Id: meta.ID, - Metadata: meta.Config.GetMetadata(), - Labels: meta.Config.GetLabels(), - Annotations: meta.Config.GetAnnotations(), - } - - if stats != nil { - var data interface{} - switch { - case typeurl.Is(stats.Data, (*v1.Metrics)(nil)): - data = &v1.Metrics{} - case typeurl.Is(stats.Data, (*v2.Metrics)(nil)): - data = &v2.Metrics{} - case typeurl.Is(stats.Data, (*wstats.Statistics)(nil)): - data = &wstats.Statistics{} - default: - return nil, errors.New("cannot convert metric data to cgroups.Metrics or windows.Statistics") - } - - if err := typeurl.UnmarshalTo(stats.Data, data); err != nil { - return nil, fmt.Errorf("failed to extract container metrics: %w", err) - } - - cpuStats, err := c.cpuContainerStats(meta.ID, false /* isSandbox */, data, protobuf.FromTimestamp(stats.Timestamp)) - if err != nil { - return nil, fmt.Errorf("failed to obtain cpu stats: %w", err) - } - cs.Cpu = cpuStats - - memoryStats, err := c.memoryContainerStats(meta.ID, data, protobuf.FromTimestamp(stats.Timestamp)) - if err != nil { - return nil, fmt.Errorf("failed to obtain memory stats: %w", err) - } - cs.Memory = memoryStats - } - - return &cs, nil -} - -// getWorkingSet calculates workingset memory from cgroup memory stats. -// The caller should make sure memory is not nil. -// workingset = usage - total_inactive_file -func getWorkingSet(memory *v1.MemoryStat) uint64 { - if memory.Usage == nil { - return 0 - } - var workingSet uint64 - if memory.TotalInactiveFile < memory.Usage.Usage { - workingSet = memory.Usage.Usage - memory.TotalInactiveFile - } - return workingSet -} - -// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats. -// The caller should make sure memory is not nil. -// workingset = usage - inactive_file -func getWorkingSetV2(memory *v2.MemoryStat) uint64 { - var workingSet uint64 - if memory.InactiveFile < memory.Usage { - workingSet = memory.Usage - memory.InactiveFile - } - return workingSet -} - -func isMemoryUnlimited(v uint64) bool { - // Size after which we consider memory to be "unlimited". This is not - // MaxInt64 due to rounding by the kernel. - // TODO: k8s or cadvisor should export this https://github.com/google/cadvisor/blob/2b6fbacac7598e0140b5bc8428e3bdd7d86cf5b9/metrics/prometheus.go#L1969-L1971 - const maxMemorySize = uint64(1 << 62) - - return v > maxMemorySize -} - -// https://github.com/kubernetes/kubernetes/blob/b47f8263e18c7b13dba33fba23187e5e0477cdbd/pkg/kubelet/stats/helper.go#L68-L71 -func getAvailableBytes(memory *v1.MemoryStat, workingSetBytes uint64) uint64 { - // memory limit - working set bytes - if !isMemoryUnlimited(memory.Usage.Limit) { - return memory.Usage.Limit - workingSetBytes - } - return 0 -} - -func getAvailableBytesV2(memory *v2.MemoryStat, workingSetBytes uint64) uint64 { - // memory limit (memory.max) for cgroupv2 - working set bytes - if !isMemoryUnlimited(memory.UsageLimit) { - return memory.UsageLimit - workingSetBytes - } - return 0 -} - -func (c *criService) cpuContainerStats(ID string, isSandbox bool, stats interface{}, timestamp time.Time) (*runtime.CpuUsage, error) { - switch metrics := stats.(type) { - case *v1.Metrics: - if metrics.CPU != nil && metrics.CPU.Usage != nil { - return &runtime.CpuUsage{ - Timestamp: timestamp.UnixNano(), - UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total}, - }, nil - } - case *v2.Metrics: - if metrics.CPU != nil { - // convert to nano seconds - usageCoreNanoSeconds := metrics.CPU.UsageUsec * 1000 - - return &runtime.CpuUsage{ - Timestamp: timestamp.UnixNano(), - UsageCoreNanoSeconds: &runtime.UInt64Value{Value: usageCoreNanoSeconds}, - }, nil - } - default: - return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath()) - } - return nil, nil -} - -func (c *criService) memoryContainerStats(ID string, stats interface{}, timestamp time.Time) (*runtime.MemoryUsage, error) { - switch metrics := stats.(type) { - case *v1.Metrics: - if metrics.Memory != nil && metrics.Memory.Usage != nil { - workingSetBytes := getWorkingSet(metrics.Memory) - - return &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{ - Value: workingSetBytes, - }, - AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytes(metrics.Memory, workingSetBytes)}, - UsageBytes: &runtime.UInt64Value{Value: metrics.Memory.Usage.Usage}, - RssBytes: &runtime.UInt64Value{Value: metrics.Memory.TotalRSS}, - PageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgFault}, - MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgMajFault}, - }, nil - } - case *v2.Metrics: - if metrics.Memory != nil { - workingSetBytes := getWorkingSetV2(metrics.Memory) - - return &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{ - Value: workingSetBytes, - }, - AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytesV2(metrics.Memory, workingSetBytes)}, - UsageBytes: &runtime.UInt64Value{Value: metrics.Memory.Usage}, - // Use Anon memory for RSS as cAdvisor on cgroupv2 - // see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799 - RssBytes: &runtime.UInt64Value{Value: metrics.Memory.Anon}, - PageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgfault}, - MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgmajfault}, - }, nil - } - default: - return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath()) - } - return nil, nil -} diff --git a/pkg/cri/sbserver/container_stats_list_linux_test.go b/pkg/cri/sbserver/container_stats_list_linux_test.go deleted file mode 100644 index 2b124cf81..000000000 --- a/pkg/cri/sbserver/container_stats_list_linux_test.go +++ /dev/null @@ -1,281 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "math" - "testing" - "time" - - v1 "github.com/containerd/cgroups/v3/cgroup1/stats" - v2 "github.com/containerd/cgroups/v3/cgroup2/stats" - "github.com/stretchr/testify/assert" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" -) - -func TestGetWorkingSet(t *testing.T) { - for _, test := range []struct { - desc string - memory *v1.MemoryStat - expected uint64 - }{ - { - desc: "nil memory usage", - memory: &v1.MemoryStat{}, - expected: 0, - }, - { - desc: "memory usage higher than inactive_total_file", - memory: &v1.MemoryStat{ - TotalInactiveFile: 1000, - Usage: &v1.MemoryEntry{Usage: 2000}, - }, - expected: 1000, - }, - { - desc: "memory usage lower than inactive_total_file", - memory: &v1.MemoryStat{ - TotalInactiveFile: 2000, - Usage: &v1.MemoryEntry{Usage: 1000}, - }, - expected: 0, - }, - } { - test := test - t.Run(test.desc, func(t *testing.T) { - got := getWorkingSet(test.memory) - assert.Equal(t, test.expected, got) - }) - } -} - -func TestGetWorkingSetV2(t *testing.T) { - for _, test := range []struct { - desc string - memory *v2.MemoryStat - expected uint64 - }{ - { - desc: "nil memory usage", - memory: &v2.MemoryStat{}, - expected: 0, - }, - { - desc: "memory usage higher than inactive_total_file", - memory: &v2.MemoryStat{ - InactiveFile: 1000, - Usage: 2000, - }, - expected: 1000, - }, - { - desc: "memory usage lower than inactive_total_file", - memory: &v2.MemoryStat{ - InactiveFile: 2000, - Usage: 1000, - }, - expected: 0, - }, - } { - test := test - t.Run(test.desc, func(t *testing.T) { - got := getWorkingSetV2(test.memory) - assert.Equal(t, test.expected, got) - }) - } -} - -func TestGetAvailableBytes(t *testing.T) { - for _, test := range []struct { - desc string - memory *v1.MemoryStat - workingSetBytes uint64 - expected uint64 - }{ - { - desc: "no limit", - memory: &v1.MemoryStat{ - Usage: &v1.MemoryEntry{ - Limit: math.MaxUint64, // no limit - Usage: 1000, - }, - }, - workingSetBytes: 500, - expected: 0, - }, - { - desc: "with limit", - memory: &v1.MemoryStat{ - Usage: &v1.MemoryEntry{ - Limit: 5000, - Usage: 1000, - }, - }, - workingSetBytes: 500, - expected: 5000 - 500, - }, - } { - test := test - t.Run(test.desc, func(t *testing.T) { - got := getAvailableBytes(test.memory, test.workingSetBytes) - assert.Equal(t, test.expected, got) - }) - } -} - -func TestGetAvailableBytesV2(t *testing.T) { - for _, test := range []struct { - desc string - memory *v2.MemoryStat - workingSetBytes uint64 - expected uint64 - }{ - { - desc: "no limit", - memory: &v2.MemoryStat{ - UsageLimit: math.MaxUint64, // no limit - Usage: 1000, - }, - workingSetBytes: 500, - expected: 0, - }, - { - desc: "with limit", - memory: &v2.MemoryStat{ - UsageLimit: 5000, - Usage: 1000, - }, - workingSetBytes: 500, - expected: 5000 - 500, - }, - } { - test := test - t.Run(test.desc, func(t *testing.T) { - got := getAvailableBytesV2(test.memory, test.workingSetBytes) - assert.Equal(t, test.expected, got) - }) - } -} - -func TestContainerMetricsMemory(t *testing.T) { - c := newTestCRIService() - timestamp := time.Now() - - for _, test := range []struct { - desc string - metrics interface{} - expected *runtime.MemoryUsage - }{ - { - desc: "v1 metrics - no memory limit", - metrics: &v1.Metrics{ - Memory: &v1.MemoryStat{ - Usage: &v1.MemoryEntry{ - Limit: math.MaxUint64, // no limit - Usage: 1000, - }, - TotalRSS: 10, - TotalPgFault: 11, - TotalPgMajFault: 12, - TotalInactiveFile: 500, - }, - }, - expected: &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{Value: 500}, - AvailableBytes: &runtime.UInt64Value{Value: 0}, - UsageBytes: &runtime.UInt64Value{Value: 1000}, - RssBytes: &runtime.UInt64Value{Value: 10}, - PageFaults: &runtime.UInt64Value{Value: 11}, - MajorPageFaults: &runtime.UInt64Value{Value: 12}, - }, - }, - { - desc: "v1 metrics - memory limit", - metrics: &v1.Metrics{ - Memory: &v1.MemoryStat{ - Usage: &v1.MemoryEntry{ - Limit: 5000, - Usage: 1000, - }, - TotalRSS: 10, - TotalPgFault: 11, - TotalPgMajFault: 12, - TotalInactiveFile: 500, - }, - }, - expected: &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{Value: 500}, - AvailableBytes: &runtime.UInt64Value{Value: 4500}, - UsageBytes: &runtime.UInt64Value{Value: 1000}, - RssBytes: &runtime.UInt64Value{Value: 10}, - PageFaults: &runtime.UInt64Value{Value: 11}, - MajorPageFaults: &runtime.UInt64Value{Value: 12}, - }, - }, - { - desc: "v2 metrics - memory limit", - metrics: &v2.Metrics{ - Memory: &v2.MemoryStat{ - Usage: 1000, - UsageLimit: 5000, - InactiveFile: 0, - Pgfault: 11, - Pgmajfault: 12, - }, - }, - expected: &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{Value: 1000}, - AvailableBytes: &runtime.UInt64Value{Value: 4000}, - UsageBytes: &runtime.UInt64Value{Value: 1000}, - RssBytes: &runtime.UInt64Value{Value: 0}, - PageFaults: &runtime.UInt64Value{Value: 11}, - MajorPageFaults: &runtime.UInt64Value{Value: 12}, - }, - }, - { - desc: "v2 metrics - no memory limit", - metrics: &v2.Metrics{ - Memory: &v2.MemoryStat{ - Usage: 1000, - UsageLimit: math.MaxUint64, // no limit - InactiveFile: 0, - Pgfault: 11, - Pgmajfault: 12, - }, - }, - expected: &runtime.MemoryUsage{ - Timestamp: timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{Value: 1000}, - AvailableBytes: &runtime.UInt64Value{Value: 0}, - UsageBytes: &runtime.UInt64Value{Value: 1000}, - RssBytes: &runtime.UInt64Value{Value: 0}, - PageFaults: &runtime.UInt64Value{Value: 11}, - MajorPageFaults: &runtime.UInt64Value{Value: 12}, - }, - }, - } { - test := test - t.Run(test.desc, func(t *testing.T) { - got, err := c.memoryContainerStats("ID", test.metrics, timestamp) - assert.NoError(t, err) - assert.Equal(t, test.expected, got) - }) - } -} diff --git a/pkg/cri/sbserver/container_stats_list_other.go b/pkg/cri/sbserver/container_stats_list_other.go deleted file mode 100644 index b92e5c361..000000000 --- a/pkg/cri/sbserver/container_stats_list_other.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build !windows && !linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "fmt" - - "github.com/containerd/containerd/api/types" - "github.com/containerd/containerd/errdefs" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - containerstore "github.com/containerd/containerd/pkg/cri/store/container" -) - -func (c *criService) containerMetrics( - meta containerstore.Metadata, - stats *types.Metric, -) (*runtime.ContainerStats, error) { - var cs runtime.ContainerStats - return &cs, fmt.Errorf("container metrics: %w", errdefs.ErrNotImplemented) -} diff --git a/pkg/cri/sbserver/container_stats_list_test.go b/pkg/cri/sbserver/container_stats_list_test.go index 0ff2e7b06..8766504fc 100644 --- a/pkg/cri/sbserver/container_stats_list_test.go +++ b/pkg/cri/sbserver/container_stats_list_test.go @@ -17,11 +17,15 @@ package sbserver import ( + "math" "testing" "time" + v1 "github.com/containerd/cgroups/v3/cgroup1/stats" + v2 "github.com/containerd/cgroups/v3/cgroup2/stats" containerstore "github.com/containerd/containerd/pkg/cri/store/container" "github.com/stretchr/testify/assert" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func TestContainerMetricsCPUNanoCoreUsage(t *testing.T) { @@ -74,3 +78,256 @@ func TestContainerMetricsCPUNanoCoreUsage(t *testing.T) { }) } } + +func TestGetWorkingSet(t *testing.T) { + for _, test := range []struct { + desc string + memory *v1.MemoryStat + expected uint64 + }{ + { + desc: "nil memory usage", + memory: &v1.MemoryStat{}, + expected: 0, + }, + { + desc: "memory usage higher than inactive_total_file", + memory: &v1.MemoryStat{ + TotalInactiveFile: 1000, + Usage: &v1.MemoryEntry{Usage: 2000}, + }, + expected: 1000, + }, + { + desc: "memory usage lower than inactive_total_file", + memory: &v1.MemoryStat{ + TotalInactiveFile: 2000, + Usage: &v1.MemoryEntry{Usage: 1000}, + }, + expected: 0, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + got := getWorkingSet(test.memory) + assert.Equal(t, test.expected, got) + }) + } +} + +func TestGetWorkingSetV2(t *testing.T) { + for _, test := range []struct { + desc string + memory *v2.MemoryStat + expected uint64 + }{ + { + desc: "nil memory usage", + memory: &v2.MemoryStat{}, + expected: 0, + }, + { + desc: "memory usage higher than inactive_total_file", + memory: &v2.MemoryStat{ + InactiveFile: 1000, + Usage: 2000, + }, + expected: 1000, + }, + { + desc: "memory usage lower than inactive_total_file", + memory: &v2.MemoryStat{ + InactiveFile: 2000, + Usage: 1000, + }, + expected: 0, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + got := getWorkingSetV2(test.memory) + assert.Equal(t, test.expected, got) + }) + } +} + +func TestGetAvailableBytes(t *testing.T) { + for _, test := range []struct { + desc string + memory *v1.MemoryStat + workingSetBytes uint64 + expected uint64 + }{ + { + desc: "no limit", + memory: &v1.MemoryStat{ + Usage: &v1.MemoryEntry{ + Limit: math.MaxUint64, // no limit + Usage: 1000, + }, + }, + workingSetBytes: 500, + expected: 0, + }, + { + desc: "with limit", + memory: &v1.MemoryStat{ + Usage: &v1.MemoryEntry{ + Limit: 5000, + Usage: 1000, + }, + }, + workingSetBytes: 500, + expected: 5000 - 500, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + got := getAvailableBytes(test.memory, test.workingSetBytes) + assert.Equal(t, test.expected, got) + }) + } +} + +func TestGetAvailableBytesV2(t *testing.T) { + for _, test := range []struct { + desc string + memory *v2.MemoryStat + workingSetBytes uint64 + expected uint64 + }{ + { + desc: "no limit", + memory: &v2.MemoryStat{ + UsageLimit: math.MaxUint64, // no limit + Usage: 1000, + }, + workingSetBytes: 500, + expected: 0, + }, + { + desc: "with limit", + memory: &v2.MemoryStat{ + UsageLimit: 5000, + Usage: 1000, + }, + workingSetBytes: 500, + expected: 5000 - 500, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + got := getAvailableBytesV2(test.memory, test.workingSetBytes) + assert.Equal(t, test.expected, got) + }) + } +} + +func TestContainerMetricsMemory(t *testing.T) { + c := newTestCRIService() + timestamp := time.Now() + + for _, test := range []struct { + desc string + metrics interface{} + expected *runtime.MemoryUsage + }{ + { + desc: "v1 metrics - no memory limit", + metrics: &v1.Metrics{ + Memory: &v1.MemoryStat{ + Usage: &v1.MemoryEntry{ + Limit: math.MaxUint64, // no limit + Usage: 1000, + }, + TotalRSS: 10, + TotalPgFault: 11, + TotalPgMajFault: 12, + TotalInactiveFile: 500, + }, + }, + expected: &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{Value: 500}, + AvailableBytes: &runtime.UInt64Value{Value: 0}, + UsageBytes: &runtime.UInt64Value{Value: 1000}, + RssBytes: &runtime.UInt64Value{Value: 10}, + PageFaults: &runtime.UInt64Value{Value: 11}, + MajorPageFaults: &runtime.UInt64Value{Value: 12}, + }, + }, + { + desc: "v1 metrics - memory limit", + metrics: &v1.Metrics{ + Memory: &v1.MemoryStat{ + Usage: &v1.MemoryEntry{ + Limit: 5000, + Usage: 1000, + }, + TotalRSS: 10, + TotalPgFault: 11, + TotalPgMajFault: 12, + TotalInactiveFile: 500, + }, + }, + expected: &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{Value: 500}, + AvailableBytes: &runtime.UInt64Value{Value: 4500}, + UsageBytes: &runtime.UInt64Value{Value: 1000}, + RssBytes: &runtime.UInt64Value{Value: 10}, + PageFaults: &runtime.UInt64Value{Value: 11}, + MajorPageFaults: &runtime.UInt64Value{Value: 12}, + }, + }, + { + desc: "v2 metrics - memory limit", + metrics: &v2.Metrics{ + Memory: &v2.MemoryStat{ + Usage: 1000, + UsageLimit: 5000, + InactiveFile: 0, + Pgfault: 11, + Pgmajfault: 12, + }, + }, + expected: &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{Value: 1000}, + AvailableBytes: &runtime.UInt64Value{Value: 4000}, + UsageBytes: &runtime.UInt64Value{Value: 1000}, + RssBytes: &runtime.UInt64Value{Value: 0}, + PageFaults: &runtime.UInt64Value{Value: 11}, + MajorPageFaults: &runtime.UInt64Value{Value: 12}, + }, + }, + { + desc: "v2 metrics - no memory limit", + metrics: &v2.Metrics{ + Memory: &v2.MemoryStat{ + Usage: 1000, + UsageLimit: math.MaxUint64, // no limit + InactiveFile: 0, + Pgfault: 11, + Pgmajfault: 12, + }, + }, + expected: &runtime.MemoryUsage{ + Timestamp: timestamp.UnixNano(), + WorkingSetBytes: &runtime.UInt64Value{Value: 1000}, + AvailableBytes: &runtime.UInt64Value{Value: 0}, + UsageBytes: &runtime.UInt64Value{Value: 1000}, + RssBytes: &runtime.UInt64Value{Value: 0}, + PageFaults: &runtime.UInt64Value{Value: 11}, + MajorPageFaults: &runtime.UInt64Value{Value: 12}, + }, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + got, err := c.memoryContainerStats("ID", test.metrics, timestamp) + assert.NoError(t, err) + assert.Equal(t, test.expected, got) + }) + } +} diff --git a/pkg/cri/sbserver/container_stats_list_windows.go b/pkg/cri/sbserver/container_stats_list_windows.go deleted file mode 100644 index c73ff4b3d..000000000 --- a/pkg/cri/sbserver/container_stats_list_windows.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sbserver - -import ( - "errors" - "fmt" - - wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" - "github.com/containerd/containerd/api/types" - "github.com/containerd/typeurl/v2" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - containerstore "github.com/containerd/containerd/pkg/cri/store/container" -) - -func (c *criService) containerMetrics( - meta containerstore.Metadata, - stats *types.Metric, -) (*runtime.ContainerStats, error) { - var cs runtime.ContainerStats - var usedBytes, inodesUsed uint64 - sn, err := c.GetSnapshot(meta.ID) - // If snapshotstore doesn't have cached snapshot information - // set WritableLayer usage to zero - if err == nil { - usedBytes = sn.Size - inodesUsed = sn.Inodes - } - cs.WritableLayer = &runtime.FilesystemUsage{ - Timestamp: sn.Timestamp, - FsId: &runtime.FilesystemIdentifier{ - Mountpoint: c.imageFSPath, - }, - UsedBytes: &runtime.UInt64Value{Value: usedBytes}, - InodesUsed: &runtime.UInt64Value{Value: inodesUsed}, - } - cs.Attributes = &runtime.ContainerAttributes{ - Id: meta.ID, - Metadata: meta.Config.GetMetadata(), - Labels: meta.Config.GetLabels(), - Annotations: meta.Config.GetAnnotations(), - } - - if stats != nil { - s, err := typeurl.UnmarshalAny(stats.Data) - if err != nil { - return nil, fmt.Errorf("failed to extract container metrics: %w", err) - } - wstats := s.(*wstats.Statistics).GetWindows() - if wstats == nil { - return nil, errors.New("windows stats is empty") - } - if wstats.Processor != nil { - cs.Cpu = &runtime.CpuUsage{ - Timestamp: wstats.Timestamp.UnixNano(), - UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS}, - } - } - if wstats.Memory != nil { - cs.Memory = &runtime.MemoryUsage{ - Timestamp: wstats.Timestamp.UnixNano(), - WorkingSetBytes: &runtime.UInt64Value{ - Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes, - }, - } - } - } - return &cs, nil -} diff --git a/pkg/cri/sbserver/sandbox_stats_windows.go b/pkg/cri/sbserver/sandbox_stats_windows.go index 0dea1f630..88162a6f1 100644 --- a/pkg/cri/sbserver/sandbox_stats_windows.go +++ b/pkg/cri/sbserver/sandbox_stats_windows.go @@ -36,7 +36,8 @@ import ( func (c *criService) podSandboxStats( ctx context.Context, - sandbox sandboxstore.Sandbox) (*runtime.PodSandboxStats, error) { + sandbox sandboxstore.Sandbox, +) (*runtime.PodSandboxStats, error) { meta := sandbox.Metadata if sandbox.Status.Get().State != sandboxstore.StateReady {