Merge pull request #8552 from dcantah/cross-plat-stats

CRI: Make stats respect sandbox's platform

pkg/cri/sbserver/container_stats.go

@@ -40,7 +40,12 @@ func (c *criService) ContainerStats(ctx context.Context, in *runtime.ContainerSt
 		return nil, fmt.Errorf("unexpected metrics response: %+v", resp.Metrics)
 	}
 
-	cs, err := c.containerMetrics(cntr.Metadata, resp.Metrics[0])
+	handler, err := c.getMetricsHandler(ctx, cntr.SandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	cs, err := handler(cntr.Metadata, resp.Metrics[0])
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode container metrics: %w", err)
 	}

pkg/cri/sbserver/container_stats_list.go

@@ -18,12 +18,20 @@ package sbserver
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"reflect"
 	"time"
 
+	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
+	cg1 "github.com/containerd/cgroups/v3/cgroup1/stats"
+	cg2 "github.com/containerd/cgroups/v3/cgroup2/stats"
 	"github.com/containerd/containerd/api/services/tasks/v1"
 	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/pkg/cri/store/stats"
+	"github.com/containerd/containerd/protobuf"
+	"github.com/containerd/typeurl/v2"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
@@ -42,14 +50,48 @@ func (c *criService) ListContainerStats(
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch metrics for tasks: %w", err)
 	}
-	criStats, err := c.toCRIContainerStats(resp.Metrics, containers)
+	criStats, err := c.toCRIContainerStats(ctx, resp.Metrics, containers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert to cri containerd stats format: %w", err)
 	}
 	return criStats, nil
 }
 
+type metricsHandler func(containerstore.Metadata, *types.Metric) (*runtime.ContainerStats, error)
+
+// Returns a function to be used for transforming container metrics into the right format.
+// Uses the platform the given sandbox advertises to implement its logic. If the platform is
+// unsupported for metrics this will return a wrapped [errdefs.ErrNotImplemented].
+func (c *criService) getMetricsHandler(ctx context.Context, sandboxID string) (metricsHandler, error) {
+	sandbox, err := c.sandboxStore.Get(sandboxID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find sandbox id %q: %w", sandboxID, err)
+	}
+	controller, err := c.getSandboxController(sandbox.Config, sandbox.RuntimeHandler)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox controller: %w", err)
+	}
+	// Grab the platform that this containers sandbox advertises. Reason being, even if
+	// the host may be {insert platform}, if it virtualizes or emulates a different platform
+	// it will return stats in that format, and we need to handle the conversion logic based
+	// off of this info.
+	p, err := controller.Platform(ctx, sandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	switch p.OS {
+	case "windows":
+		return c.windowsContainerMetrics, nil
+	case "linux":
+		return c.linuxContainerMetrics, nil
+	default:
+		return nil, fmt.Errorf("container metrics for platform %+v: %w", p, errdefs.ErrNotImplemented)
+	}
+}
+
 func (c *criService) toCRIContainerStats(
+	ctx context.Context,
 	stats []*types.Metric,
 	containers []containerstore.Container,
 ) (*runtime.ListContainerStatsResponse, error) {
@@ -58,8 +100,29 @@ func (c *criService) toCRIContainerStats(
 		statsMap[stat.ID] = stat
 	}
 	containerStats := new(runtime.ListContainerStatsResponse)
+
+	// Unfortunately if no filter was passed we're asking for every containers stats which
+	// generally belong to multiple different pods, who all might have different platforms.
+	// To avoid recalculating the right metricsHandler to invoke, if we've already calculated
+	// the platform and handler for a given sandbox just pull it from our map here.
+	var (
+		err     error
+		handler metricsHandler
+	)
+	sandboxToMetricsHandler := make(map[string]metricsHandler)
 	for _, cntr := range containers {
-		cs, err := c.containerMetrics(cntr.Metadata, statsMap[cntr.ID])
+		h, ok := sandboxToMetricsHandler[cntr.SandboxID]
+		if !ok {
+			handler, err = c.getMetricsHandler(ctx, cntr.SandboxID)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get metrics handler for container %q: %w", cntr.ID, err)
+			}
+			sandboxToMetricsHandler[cntr.SandboxID] = handler
+		} else {
+			handler = h
+		}
+
+		cs, err := handler(cntr.Metadata, statsMap[cntr.ID])
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode container metrics for %q: %w", cntr.ID, err)
 		}
@@ -72,7 +135,6 @@ func (c *criService) toCRIContainerStats(
 			}
 			cs.Cpu.UsageNanoCores = &runtime.UInt64Value{Value: nanoUsage}
 		}
-
 		containerStats.Stats = append(containerStats.Stats, cs)
 	}
 	return containerStats, nil
@@ -133,7 +195,6 @@ func (c *criService) getUsageNanoCores(containerID string, isSandbox bool, curre
 		if err != nil {
 			return 0, fmt.Errorf("failed to update sandbox container stats: %s: %w", containerID, err)
 		}
-
 	} else {
 		err := c.containerStore.UpdateContainerStats(containerID, newStats)
 		if err != nil {
@@ -193,3 +254,238 @@ func matchLabelSelector(selector, labels map[string]string) bool {
 	}
 	return true
 }
+
+func (c *criService) windowsContainerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.GetSnapshot(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		s, err := typeurl.UnmarshalAny(stats.Data)
+		if err != nil {
+			return nil, fmt.Errorf("failed to extract container metrics: %w", err)
+		}
+		wstats := s.(*wstats.Statistics).GetWindows()
+		if wstats == nil {
+			return nil, errors.New("windows stats is empty")
+		}
+		if wstats.Processor != nil {
+			cs.Cpu = &runtime.CpuUsage{
+				Timestamp:            wstats.Timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS},
+			}
+		}
+		if wstats.Memory != nil {
+			cs.Memory = &runtime.MemoryUsage{
+				Timestamp: wstats.Timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes,
+				},
+			}
+		}
+	}
+	return &cs, nil
+}
+
+func (c *criService) linuxContainerMetrics(
+	meta containerstore.Metadata,
+	stats *types.Metric,
+) (*runtime.ContainerStats, error) {
+	var cs runtime.ContainerStats
+	var usedBytes, inodesUsed uint64
+	sn, err := c.GetSnapshot(meta.ID)
+	// If snapshotstore doesn't have cached snapshot information
+	// set WritableLayer usage to zero
+	if err == nil {
+		usedBytes = sn.Size
+		inodesUsed = sn.Inodes
+	}
+	cs.WritableLayer = &runtime.FilesystemUsage{
+		Timestamp: sn.Timestamp,
+		FsId: &runtime.FilesystemIdentifier{
+			Mountpoint: c.imageFSPath,
+		},
+		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
+		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
+	}
+	cs.Attributes = &runtime.ContainerAttributes{
+		Id:          meta.ID,
+		Metadata:    meta.Config.GetMetadata(),
+		Labels:      meta.Config.GetLabels(),
+		Annotations: meta.Config.GetAnnotations(),
+	}
+
+	if stats != nil {
+		var data interface{}
+		switch {
+		case typeurl.Is(stats.Data, (*cg1.Metrics)(nil)):
+			data = &cg1.Metrics{}
+		case typeurl.Is(stats.Data, (*cg2.Metrics)(nil)):
+			data = &cg2.Metrics{}
+		case typeurl.Is(stats.Data, (*wstats.Statistics)(nil)):
+			data = &wstats.Statistics{}
+		default:
+			return nil, errors.New("cannot convert metric data to cgroups.Metrics or windows.Statistics")
+		}
+
+		if err := typeurl.UnmarshalTo(stats.Data, data); err != nil {
+			return nil, fmt.Errorf("failed to extract container metrics: %w", err)
+		}
+
+		cpuStats, err := c.cpuContainerStats(meta.ID, false /* isSandbox */, data, protobuf.FromTimestamp(stats.Timestamp))
+		if err != nil {
+			return nil, fmt.Errorf("failed to obtain cpu stats: %w", err)
+		}
+		cs.Cpu = cpuStats
+
+		memoryStats, err := c.memoryContainerStats(meta.ID, data, protobuf.FromTimestamp(stats.Timestamp))
+		if err != nil {
+			return nil, fmt.Errorf("failed to obtain memory stats: %w", err)
+		}
+		cs.Memory = memoryStats
+	}
+
+	return &cs, nil
+}
+
+// getWorkingSet calculates workingset memory from cgroup memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - total_inactive_file
+func getWorkingSet(memory *cg1.MemoryStat) uint64 {
+	if memory.Usage == nil {
+		return 0
+	}
+	var workingSet uint64
+	if memory.TotalInactiveFile < memory.Usage.Usage {
+		workingSet = memory.Usage.Usage - memory.TotalInactiveFile
+	}
+	return workingSet
+}
+
+// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats.
+// The caller should make sure memory is not nil.
+// workingset = usage - inactive_file
+func getWorkingSetV2(memory *cg2.MemoryStat) uint64 {
+	var workingSet uint64
+	if memory.InactiveFile < memory.Usage {
+		workingSet = memory.Usage - memory.InactiveFile
+	}
+	return workingSet
+}
+
+func isMemoryUnlimited(v uint64) bool {
+	// Size after which we consider memory to be "unlimited". This is not
+	// MaxInt64 due to rounding by the kernel.
+	// TODO: k8s or cadvisor should export this https://github.com/google/cadvisor/blob/2b6fbacac7598e0140b5bc8428e3bdd7d86cf5b9/metrics/prometheus.go#L1969-L1971
+	const maxMemorySize = uint64(1 << 62)
+
+	return v > maxMemorySize
+}
+
+// https://github.com/kubernetes/kubernetes/blob/b47f8263e18c7b13dba33fba23187e5e0477cdbd/pkg/kubelet/stats/helper.go#L68-L71
+func getAvailableBytes(memory *cg1.MemoryStat, workingSetBytes uint64) uint64 {
+	// memory limit - working set bytes
+	if !isMemoryUnlimited(memory.Usage.Limit) {
+		return memory.Usage.Limit - workingSetBytes
+	}
+	return 0
+}
+
+func getAvailableBytesV2(memory *cg2.MemoryStat, workingSetBytes uint64) uint64 {
+	// memory limit (memory.max) for cgroupv2 - working set bytes
+	if !isMemoryUnlimited(memory.UsageLimit) {
+		return memory.UsageLimit - workingSetBytes
+	}
+	return 0
+}
+
+func (c *criService) cpuContainerStats(ID string, isSandbox bool, stats interface{}, timestamp time.Time) (*runtime.CpuUsage, error) {
+	switch metrics := stats.(type) {
+	case *cg1.Metrics:
+		metrics.GetCPU().GetUsage()
+		if metrics.CPU != nil && metrics.CPU.Usage != nil {
+			return &runtime.CpuUsage{
+				Timestamp:            timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total},
+			}, nil
+		}
+	case *cg2.Metrics:
+		if metrics.CPU != nil {
+			// convert to nano seconds
+			usageCoreNanoSeconds := metrics.CPU.UsageUsec * 1000
+
+			return &runtime.CpuUsage{
+				Timestamp:            timestamp.UnixNano(),
+				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: usageCoreNanoSeconds},
+			}, nil
+		}
+	default:
+		return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath())
+	}
+	return nil, nil
+}
+
+func (c *criService) memoryContainerStats(ID string, stats interface{}, timestamp time.Time) (*runtime.MemoryUsage, error) {
+	switch metrics := stats.(type) {
+	case *cg1.Metrics:
+		if metrics.Memory != nil && metrics.Memory.Usage != nil {
+			workingSetBytes := getWorkingSet(metrics.Memory)
+
+			return &runtime.MemoryUsage{
+				Timestamp: timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: workingSetBytes,
+				},
+				AvailableBytes:  &runtime.UInt64Value{Value: getAvailableBytes(metrics.Memory, workingSetBytes)},
+				UsageBytes:      &runtime.UInt64Value{Value: metrics.Memory.Usage.Usage},
+				RssBytes:        &runtime.UInt64Value{Value: metrics.Memory.TotalRSS},
+				PageFaults:      &runtime.UInt64Value{Value: metrics.Memory.TotalPgFault},
+				MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgMajFault},
+			}, nil
+		}
+	case *cg2.Metrics:
+		if metrics.Memory != nil {
+			workingSetBytes := getWorkingSetV2(metrics.Memory)
+
+			return &runtime.MemoryUsage{
+				Timestamp: timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{
+					Value: workingSetBytes,
+				},
+				AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytesV2(metrics.Memory, workingSetBytes)},
+				UsageBytes:     &runtime.UInt64Value{Value: metrics.Memory.Usage},
+				// Use Anon memory for RSS as cAdvisor on cgroupv2
+				// see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799
+				RssBytes:        &runtime.UInt64Value{Value: metrics.Memory.Anon},
+				PageFaults:      &runtime.UInt64Value{Value: metrics.Memory.Pgfault},
+				MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgmajfault},
+			}, nil
+		}
+	default:
+		return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath())
+	}
+	return nil, nil
+}

pkg/cri/sbserver/container_stats_list_linux.go

@@ -1,213 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sbserver
-
-import (
-	"errors"
-	"fmt"
-	"reflect"
-	"time"
-
-	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
-	v1 "github.com/containerd/cgroups/v3/cgroup1/stats"
-	v2 "github.com/containerd/cgroups/v3/cgroup2/stats"
-	"github.com/containerd/containerd/api/types"
-	"github.com/containerd/containerd/protobuf"
-	"github.com/containerd/typeurl/v2"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
-)
-
-func (c *criService) containerMetrics(
-	meta containerstore.Metadata,
-	stats *types.Metric,
-) (*runtime.ContainerStats, error) {
-	var cs runtime.ContainerStats
-	var usedBytes, inodesUsed uint64
-	sn, err := c.GetSnapshot(meta.ID)
-	// If snapshotstore doesn't have cached snapshot information
-	// set WritableLayer usage to zero
-	if err == nil {
-		usedBytes = sn.Size
-		inodesUsed = sn.Inodes
-	}
-	cs.WritableLayer = &runtime.FilesystemUsage{
-		Timestamp: sn.Timestamp,
-		FsId: &runtime.FilesystemIdentifier{
-			Mountpoint: c.imageFSPath,
-		},
-		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
-		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
-	}
-	cs.Attributes = &runtime.ContainerAttributes{
-		Id:          meta.ID,
-		Metadata:    meta.Config.GetMetadata(),
-		Labels:      meta.Config.GetLabels(),
-		Annotations: meta.Config.GetAnnotations(),
-	}
-
-	if stats != nil {
-		var data interface{}
-		switch {
-		case typeurl.Is(stats.Data, (*v1.Metrics)(nil)):
-			data = &v1.Metrics{}
-		case typeurl.Is(stats.Data, (*v2.Metrics)(nil)):
-			data = &v2.Metrics{}
-		case typeurl.Is(stats.Data, (*wstats.Statistics)(nil)):
-			data = &wstats.Statistics{}
-		default:
-			return nil, errors.New("cannot convert metric data to cgroups.Metrics or windows.Statistics")
-		}
-
-		if err := typeurl.UnmarshalTo(stats.Data, data); err != nil {
-			return nil, fmt.Errorf("failed to extract container metrics: %w", err)
-		}
-
-		cpuStats, err := c.cpuContainerStats(meta.ID, false /* isSandbox */, data, protobuf.FromTimestamp(stats.Timestamp))
-		if err != nil {
-			return nil, fmt.Errorf("failed to obtain cpu stats: %w", err)
-		}
-		cs.Cpu = cpuStats
-
-		memoryStats, err := c.memoryContainerStats(meta.ID, data, protobuf.FromTimestamp(stats.Timestamp))
-		if err != nil {
-			return nil, fmt.Errorf("failed to obtain memory stats: %w", err)
-		}
-		cs.Memory = memoryStats
-	}
-
-	return &cs, nil
-}
-
-// getWorkingSet calculates workingset memory from cgroup memory stats.
-// The caller should make sure memory is not nil.
-// workingset = usage - total_inactive_file
-func getWorkingSet(memory *v1.MemoryStat) uint64 {
-	if memory.Usage == nil {
-		return 0
-	}
-	var workingSet uint64
-	if memory.TotalInactiveFile < memory.Usage.Usage {
-		workingSet = memory.Usage.Usage - memory.TotalInactiveFile
-	}
-	return workingSet
-}
-
-// getWorkingSetV2 calculates workingset memory from cgroupv2 memory stats.
-// The caller should make sure memory is not nil.
-// workingset = usage - inactive_file
-func getWorkingSetV2(memory *v2.MemoryStat) uint64 {
-	var workingSet uint64
-	if memory.InactiveFile < memory.Usage {
-		workingSet = memory.Usage - memory.InactiveFile
-	}
-	return workingSet
-}
-
-func isMemoryUnlimited(v uint64) bool {
-	// Size after which we consider memory to be "unlimited". This is not
-	// MaxInt64 due to rounding by the kernel.
-	// TODO: k8s or cadvisor should export this https://github.com/google/cadvisor/blob/2b6fbacac7598e0140b5bc8428e3bdd7d86cf5b9/metrics/prometheus.go#L1969-L1971
-	const maxMemorySize = uint64(1 << 62)
-
-	return v > maxMemorySize
-}
-
-// https://github.com/kubernetes/kubernetes/blob/b47f8263e18c7b13dba33fba23187e5e0477cdbd/pkg/kubelet/stats/helper.go#L68-L71
-func getAvailableBytes(memory *v1.MemoryStat, workingSetBytes uint64) uint64 {
-	// memory limit - working set bytes
-	if !isMemoryUnlimited(memory.Usage.Limit) {
-		return memory.Usage.Limit - workingSetBytes
-	}
-	return 0
-}
-
-func getAvailableBytesV2(memory *v2.MemoryStat, workingSetBytes uint64) uint64 {
-	// memory limit (memory.max) for cgroupv2 - working set bytes
-	if !isMemoryUnlimited(memory.UsageLimit) {
-		return memory.UsageLimit - workingSetBytes
-	}
-	return 0
-}
-
-func (c *criService) cpuContainerStats(ID string, isSandbox bool, stats interface{}, timestamp time.Time) (*runtime.CpuUsage, error) {
-	switch metrics := stats.(type) {
-	case *v1.Metrics:
-		if metrics.CPU != nil && metrics.CPU.Usage != nil {
-			return &runtime.CpuUsage{
-				Timestamp:            timestamp.UnixNano(),
-				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: metrics.CPU.Usage.Total},
-			}, nil
-		}
-	case *v2.Metrics:
-		if metrics.CPU != nil {
-			// convert to nano seconds
-			usageCoreNanoSeconds := metrics.CPU.UsageUsec * 1000
-
-			return &runtime.CpuUsage{
-				Timestamp:            timestamp.UnixNano(),
-				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: usageCoreNanoSeconds},
-			}, nil
-		}
-	default:
-		return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath())
-	}
-	return nil, nil
-}
-
-func (c *criService) memoryContainerStats(ID string, stats interface{}, timestamp time.Time) (*runtime.MemoryUsage, error) {
-	switch metrics := stats.(type) {
-	case *v1.Metrics:
-		if metrics.Memory != nil && metrics.Memory.Usage != nil {
-			workingSetBytes := getWorkingSet(metrics.Memory)
-
-			return &runtime.MemoryUsage{
-				Timestamp: timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{
-					Value: workingSetBytes,
-				},
-				AvailableBytes:  &runtime.UInt64Value{Value: getAvailableBytes(metrics.Memory, workingSetBytes)},
-				UsageBytes:      &runtime.UInt64Value{Value: metrics.Memory.Usage.Usage},
-				RssBytes:        &runtime.UInt64Value{Value: metrics.Memory.TotalRSS},
-				PageFaults:      &runtime.UInt64Value{Value: metrics.Memory.TotalPgFault},
-				MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.TotalPgMajFault},
-			}, nil
-		}
-	case *v2.Metrics:
-		if metrics.Memory != nil {
-			workingSetBytes := getWorkingSetV2(metrics.Memory)
-
-			return &runtime.MemoryUsage{
-				Timestamp: timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{
-					Value: workingSetBytes,
-				},
-				AvailableBytes: &runtime.UInt64Value{Value: getAvailableBytesV2(metrics.Memory, workingSetBytes)},
-				UsageBytes:     &runtime.UInt64Value{Value: metrics.Memory.Usage},
-				// Use Anon memory for RSS as cAdvisor on cgroupv2
-				// see https://github.com/google/cadvisor/blob/a9858972e75642c2b1914c8d5428e33e6392c08a/container/libcontainer/handler.go#L799
-				RssBytes:        &runtime.UInt64Value{Value: metrics.Memory.Anon},
-				PageFaults:      &runtime.UInt64Value{Value: metrics.Memory.Pgfault},
-				MajorPageFaults: &runtime.UInt64Value{Value: metrics.Memory.Pgmajfault},
-			}, nil
-		}
-	default:
-		return nil, fmt.Errorf("unexpected metrics type: %T from %s", metrics, reflect.TypeOf(metrics).Elem().PkgPath())
-	}
-	return nil, nil
-}

pkg/cri/sbserver/container_stats_list_linux_test.go

@@ -1,281 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sbserver
-
-import (
-	"math"
-	"testing"
-	"time"
-
-	v1 "github.com/containerd/cgroups/v3/cgroup1/stats"
-	v2 "github.com/containerd/cgroups/v3/cgroup2/stats"
-	"github.com/stretchr/testify/assert"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-)
-
-func TestGetWorkingSet(t *testing.T) {
-	for _, test := range []struct {
-		desc     string
-		memory   *v1.MemoryStat
-		expected uint64
-	}{
-		{
-			desc:     "nil memory usage",
-			memory:   &v1.MemoryStat{},
-			expected: 0,
-		},
-		{
-			desc: "memory usage higher than inactive_total_file",
-			memory: &v1.MemoryStat{
-				TotalInactiveFile: 1000,
-				Usage:             &v1.MemoryEntry{Usage: 2000},
-			},
-			expected: 1000,
-		},
-		{
-			desc: "memory usage lower than inactive_total_file",
-			memory: &v1.MemoryStat{
-				TotalInactiveFile: 2000,
-				Usage:             &v1.MemoryEntry{Usage: 1000},
-			},
-			expected: 0,
-		},
-	} {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			got := getWorkingSet(test.memory)
-			assert.Equal(t, test.expected, got)
-		})
-	}
-}
-
-func TestGetWorkingSetV2(t *testing.T) {
-	for _, test := range []struct {
-		desc     string
-		memory   *v2.MemoryStat
-		expected uint64
-	}{
-		{
-			desc:     "nil memory usage",
-			memory:   &v2.MemoryStat{},
-			expected: 0,
-		},
-		{
-			desc: "memory usage higher than inactive_total_file",
-			memory: &v2.MemoryStat{
-				InactiveFile: 1000,
-				Usage:        2000,
-			},
-			expected: 1000,
-		},
-		{
-			desc: "memory usage lower than inactive_total_file",
-			memory: &v2.MemoryStat{
-				InactiveFile: 2000,
-				Usage:        1000,
-			},
-			expected: 0,
-		},
-	} {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			got := getWorkingSetV2(test.memory)
-			assert.Equal(t, test.expected, got)
-		})
-	}
-}
-
-func TestGetAvailableBytes(t *testing.T) {
-	for _, test := range []struct {
-		desc            string
-		memory          *v1.MemoryStat
-		workingSetBytes uint64
-		expected        uint64
-	}{
-		{
-			desc: "no limit",
-			memory: &v1.MemoryStat{
-				Usage: &v1.MemoryEntry{
-					Limit: math.MaxUint64, // no limit
-					Usage: 1000,
-				},
-			},
-			workingSetBytes: 500,
-			expected:        0,
-		},
-		{
-			desc: "with limit",
-			memory: &v1.MemoryStat{
-				Usage: &v1.MemoryEntry{
-					Limit: 5000,
-					Usage: 1000,
-				},
-			},
-			workingSetBytes: 500,
-			expected:        5000 - 500,
-		},
-	} {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			got := getAvailableBytes(test.memory, test.workingSetBytes)
-			assert.Equal(t, test.expected, got)
-		})
-	}
-}
-
-func TestGetAvailableBytesV2(t *testing.T) {
-	for _, test := range []struct {
-		desc            string
-		memory          *v2.MemoryStat
-		workingSetBytes uint64
-		expected        uint64
-	}{
-		{
-			desc: "no limit",
-			memory: &v2.MemoryStat{
-				UsageLimit: math.MaxUint64, // no limit
-				Usage:      1000,
-			},
-			workingSetBytes: 500,
-			expected:        0,
-		},
-		{
-			desc: "with limit",
-			memory: &v2.MemoryStat{
-				UsageLimit: 5000,
-				Usage:      1000,
-			},
-			workingSetBytes: 500,
-			expected:        5000 - 500,
-		},
-	} {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			got := getAvailableBytesV2(test.memory, test.workingSetBytes)
-			assert.Equal(t, test.expected, got)
-		})
-	}
-}
-
-func TestContainerMetricsMemory(t *testing.T) {
-	c := newTestCRIService()
-	timestamp := time.Now()
-
-	for _, test := range []struct {
-		desc     string
-		metrics  interface{}
-		expected *runtime.MemoryUsage
-	}{
-		{
-			desc: "v1 metrics - no memory limit",
-			metrics: &v1.Metrics{
-				Memory: &v1.MemoryStat{
-					Usage: &v1.MemoryEntry{
-						Limit: math.MaxUint64, // no limit
-						Usage: 1000,
-					},
-					TotalRSS:          10,
-					TotalPgFault:      11,
-					TotalPgMajFault:   12,
-					TotalInactiveFile: 500,
-				},
-			},
-			expected: &runtime.MemoryUsage{
-				Timestamp:       timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{Value: 500},
-				AvailableBytes:  &runtime.UInt64Value{Value: 0},
-				UsageBytes:      &runtime.UInt64Value{Value: 1000},
-				RssBytes:        &runtime.UInt64Value{Value: 10},
-				PageFaults:      &runtime.UInt64Value{Value: 11},
-				MajorPageFaults: &runtime.UInt64Value{Value: 12},
-			},
-		},
-		{
-			desc: "v1 metrics - memory limit",
-			metrics: &v1.Metrics{
-				Memory: &v1.MemoryStat{
-					Usage: &v1.MemoryEntry{
-						Limit: 5000,
-						Usage: 1000,
-					},
-					TotalRSS:          10,
-					TotalPgFault:      11,
-					TotalPgMajFault:   12,
-					TotalInactiveFile: 500,
-				},
-			},
-			expected: &runtime.MemoryUsage{
-				Timestamp:       timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{Value: 500},
-				AvailableBytes:  &runtime.UInt64Value{Value: 4500},
-				UsageBytes:      &runtime.UInt64Value{Value: 1000},
-				RssBytes:        &runtime.UInt64Value{Value: 10},
-				PageFaults:      &runtime.UInt64Value{Value: 11},
-				MajorPageFaults: &runtime.UInt64Value{Value: 12},
-			},
-		},
-		{
-			desc: "v2 metrics - memory limit",
-			metrics: &v2.Metrics{
-				Memory: &v2.MemoryStat{
-					Usage:        1000,
-					UsageLimit:   5000,
-					InactiveFile: 0,
-					Pgfault:      11,
-					Pgmajfault:   12,
-				},
-			},
-			expected: &runtime.MemoryUsage{
-				Timestamp:       timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{Value: 1000},
-				AvailableBytes:  &runtime.UInt64Value{Value: 4000},
-				UsageBytes:      &runtime.UInt64Value{Value: 1000},
-				RssBytes:        &runtime.UInt64Value{Value: 0},
-				PageFaults:      &runtime.UInt64Value{Value: 11},
-				MajorPageFaults: &runtime.UInt64Value{Value: 12},
-			},
-		},
-		{
-			desc: "v2 metrics - no memory limit",
-			metrics: &v2.Metrics{
-				Memory: &v2.MemoryStat{
-					Usage:        1000,
-					UsageLimit:   math.MaxUint64, // no limit
-					InactiveFile: 0,
-					Pgfault:      11,
-					Pgmajfault:   12,
-				},
-			},
-			expected: &runtime.MemoryUsage{
-				Timestamp:       timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{Value: 1000},
-				AvailableBytes:  &runtime.UInt64Value{Value: 0},
-				UsageBytes:      &runtime.UInt64Value{Value: 1000},
-				RssBytes:        &runtime.UInt64Value{Value: 0},
-				PageFaults:      &runtime.UInt64Value{Value: 11},
-				MajorPageFaults: &runtime.UInt64Value{Value: 12},
-			},
-		},
-	} {
-		test := test
-		t.Run(test.desc, func(t *testing.T) {
-			got, err := c.memoryContainerStats("ID", test.metrics, timestamp)
-			assert.NoError(t, err)
-			assert.Equal(t, test.expected, got)
-		})
-	}
-}

pkg/cri/sbserver/container_stats_list_other.go

@@ -1,37 +0,0 @@
-//go:build !windows && !linux
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sbserver
-
-import (
-	"fmt"
-
-	"github.com/containerd/containerd/api/types"
-	"github.com/containerd/containerd/errdefs"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
-)
-
-func (c *criService) containerMetrics(
-	meta containerstore.Metadata,
-	stats *types.Metric,
-) (*runtime.ContainerStats, error) {
-	var cs runtime.ContainerStats
-	return &cs, fmt.Errorf("container metrics: %w", errdefs.ErrNotImplemented)
-}

pkg/cri/sbserver/container_stats_list_test.go

@@ -17,11 +17,15 @@
 package sbserver
 
 import (
+	"math"
 	"testing"
 	"time"
 
+	v1 "github.com/containerd/cgroups/v3/cgroup1/stats"
+	v2 "github.com/containerd/cgroups/v3/cgroup2/stats"
 	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
 	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
 func TestContainerMetricsCPUNanoCoreUsage(t *testing.T) {
@@ -74,3 +78,256 @@ func TestContainerMetricsCPUNanoCoreUsage(t *testing.T) {
 		})
 	}
 }
+
+func TestGetWorkingSet(t *testing.T) {
+	for _, test := range []struct {
+		desc     string
+		memory   *v1.MemoryStat
+		expected uint64
+	}{
+		{
+			desc:     "nil memory usage",
+			memory:   &v1.MemoryStat{},
+			expected: 0,
+		},
+		{
+			desc: "memory usage higher than inactive_total_file",
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 1000,
+				Usage:             &v1.MemoryEntry{Usage: 2000},
+			},
+			expected: 1000,
+		},
+		{
+			desc: "memory usage lower than inactive_total_file",
+			memory: &v1.MemoryStat{
+				TotalInactiveFile: 2000,
+				Usage:             &v1.MemoryEntry{Usage: 1000},
+			},
+			expected: 0,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			got := getWorkingSet(test.memory)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}
+
+func TestGetWorkingSetV2(t *testing.T) {
+	for _, test := range []struct {
+		desc     string
+		memory   *v2.MemoryStat
+		expected uint64
+	}{
+		{
+			desc:     "nil memory usage",
+			memory:   &v2.MemoryStat{},
+			expected: 0,
+		},
+		{
+			desc: "memory usage higher than inactive_total_file",
+			memory: &v2.MemoryStat{
+				InactiveFile: 1000,
+				Usage:        2000,
+			},
+			expected: 1000,
+		},
+		{
+			desc: "memory usage lower than inactive_total_file",
+			memory: &v2.MemoryStat{
+				InactiveFile: 2000,
+				Usage:        1000,
+			},
+			expected: 0,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			got := getWorkingSetV2(test.memory)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}
+
+func TestGetAvailableBytes(t *testing.T) {
+	for _, test := range []struct {
+		desc            string
+		memory          *v1.MemoryStat
+		workingSetBytes uint64
+		expected        uint64
+	}{
+		{
+			desc: "no limit",
+			memory: &v1.MemoryStat{
+				Usage: &v1.MemoryEntry{
+					Limit: math.MaxUint64, // no limit
+					Usage: 1000,
+				},
+			},
+			workingSetBytes: 500,
+			expected:        0,
+		},
+		{
+			desc: "with limit",
+			memory: &v1.MemoryStat{
+				Usage: &v1.MemoryEntry{
+					Limit: 5000,
+					Usage: 1000,
+				},
+			},
+			workingSetBytes: 500,
+			expected:        5000 - 500,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			got := getAvailableBytes(test.memory, test.workingSetBytes)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}
+
+func TestGetAvailableBytesV2(t *testing.T) {
+	for _, test := range []struct {
+		desc            string
+		memory          *v2.MemoryStat
+		workingSetBytes uint64
+		expected        uint64
+	}{
+		{
+			desc: "no limit",
+			memory: &v2.MemoryStat{
+				UsageLimit: math.MaxUint64, // no limit
+				Usage:      1000,
+			},
+			workingSetBytes: 500,
+			expected:        0,
+		},
+		{
+			desc: "with limit",
+			memory: &v2.MemoryStat{
+				UsageLimit: 5000,
+				Usage:      1000,
+			},
+			workingSetBytes: 500,
+			expected:        5000 - 500,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			got := getAvailableBytesV2(test.memory, test.workingSetBytes)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}
+
+func TestContainerMetricsMemory(t *testing.T) {
+	c := newTestCRIService()
+	timestamp := time.Now()
+
+	for _, test := range []struct {
+		desc     string
+		metrics  interface{}
+		expected *runtime.MemoryUsage
+	}{
+		{
+			desc: "v1 metrics - no memory limit",
+			metrics: &v1.Metrics{
+				Memory: &v1.MemoryStat{
+					Usage: &v1.MemoryEntry{
+						Limit: math.MaxUint64, // no limit
+						Usage: 1000,
+					},
+					TotalRSS:          10,
+					TotalPgFault:      11,
+					TotalPgMajFault:   12,
+					TotalInactiveFile: 500,
+				},
+			},
+			expected: &runtime.MemoryUsage{
+				Timestamp:       timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{Value: 500},
+				AvailableBytes:  &runtime.UInt64Value{Value: 0},
+				UsageBytes:      &runtime.UInt64Value{Value: 1000},
+				RssBytes:        &runtime.UInt64Value{Value: 10},
+				PageFaults:      &runtime.UInt64Value{Value: 11},
+				MajorPageFaults: &runtime.UInt64Value{Value: 12},
+			},
+		},
+		{
+			desc: "v1 metrics - memory limit",
+			metrics: &v1.Metrics{
+				Memory: &v1.MemoryStat{
+					Usage: &v1.MemoryEntry{
+						Limit: 5000,
+						Usage: 1000,
+					},
+					TotalRSS:          10,
+					TotalPgFault:      11,
+					TotalPgMajFault:   12,
+					TotalInactiveFile: 500,
+				},
+			},
+			expected: &runtime.MemoryUsage{
+				Timestamp:       timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{Value: 500},
+				AvailableBytes:  &runtime.UInt64Value{Value: 4500},
+				UsageBytes:      &runtime.UInt64Value{Value: 1000},
+				RssBytes:        &runtime.UInt64Value{Value: 10},
+				PageFaults:      &runtime.UInt64Value{Value: 11},
+				MajorPageFaults: &runtime.UInt64Value{Value: 12},
+			},
+		},
+		{
+			desc: "v2 metrics - memory limit",
+			metrics: &v2.Metrics{
+				Memory: &v2.MemoryStat{
+					Usage:        1000,
+					UsageLimit:   5000,
+					InactiveFile: 0,
+					Pgfault:      11,
+					Pgmajfault:   12,
+				},
+			},
+			expected: &runtime.MemoryUsage{
+				Timestamp:       timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{Value: 1000},
+				AvailableBytes:  &runtime.UInt64Value{Value: 4000},
+				UsageBytes:      &runtime.UInt64Value{Value: 1000},
+				RssBytes:        &runtime.UInt64Value{Value: 0},
+				PageFaults:      &runtime.UInt64Value{Value: 11},
+				MajorPageFaults: &runtime.UInt64Value{Value: 12},
+			},
+		},
+		{
+			desc: "v2 metrics - no memory limit",
+			metrics: &v2.Metrics{
+				Memory: &v2.MemoryStat{
+					Usage:        1000,
+					UsageLimit:   math.MaxUint64, // no limit
+					InactiveFile: 0,
+					Pgfault:      11,
+					Pgmajfault:   12,
+				},
+			},
+			expected: &runtime.MemoryUsage{
+				Timestamp:       timestamp.UnixNano(),
+				WorkingSetBytes: &runtime.UInt64Value{Value: 1000},
+				AvailableBytes:  &runtime.UInt64Value{Value: 0},
+				UsageBytes:      &runtime.UInt64Value{Value: 1000},
+				RssBytes:        &runtime.UInt64Value{Value: 0},
+				PageFaults:      &runtime.UInt64Value{Value: 11},
+				MajorPageFaults: &runtime.UInt64Value{Value: 12},
+			},
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			got, err := c.memoryContainerStats("ID", test.metrics, timestamp)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expected, got)
+		})
+	}
+}

pkg/cri/sbserver/container_stats_list_windows.go

@@ -1,84 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sbserver
-
-import (
-	"errors"
-	"fmt"
-
-	wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats"
-	"github.com/containerd/containerd/api/types"
-	"github.com/containerd/typeurl/v2"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	containerstore "github.com/containerd/containerd/pkg/cri/store/container"
-)
-
-func (c *criService) containerMetrics(
-	meta containerstore.Metadata,
-	stats *types.Metric,
-) (*runtime.ContainerStats, error) {
-	var cs runtime.ContainerStats
-	var usedBytes, inodesUsed uint64
-	sn, err := c.GetSnapshot(meta.ID)
-	// If snapshotstore doesn't have cached snapshot information
-	// set WritableLayer usage to zero
-	if err == nil {
-		usedBytes = sn.Size
-		inodesUsed = sn.Inodes
-	}
-	cs.WritableLayer = &runtime.FilesystemUsage{
-		Timestamp: sn.Timestamp,
-		FsId: &runtime.FilesystemIdentifier{
-			Mountpoint: c.imageFSPath,
-		},
-		UsedBytes:  &runtime.UInt64Value{Value: usedBytes},
-		InodesUsed: &runtime.UInt64Value{Value: inodesUsed},
-	}
-	cs.Attributes = &runtime.ContainerAttributes{
-		Id:          meta.ID,
-		Metadata:    meta.Config.GetMetadata(),
-		Labels:      meta.Config.GetLabels(),
-		Annotations: meta.Config.GetAnnotations(),
-	}
-
-	if stats != nil {
-		s, err := typeurl.UnmarshalAny(stats.Data)
-		if err != nil {
-			return nil, fmt.Errorf("failed to extract container metrics: %w", err)
-		}
-		wstats := s.(*wstats.Statistics).GetWindows()
-		if wstats == nil {
-			return nil, errors.New("windows stats is empty")
-		}
-		if wstats.Processor != nil {
-			cs.Cpu = &runtime.CpuUsage{
-				Timestamp:            wstats.Timestamp.UnixNano(),
-				UsageCoreNanoSeconds: &runtime.UInt64Value{Value: wstats.Processor.TotalRuntimeNS},
-			}
-		}
-		if wstats.Memory != nil {
-			cs.Memory = &runtime.MemoryUsage{
-				Timestamp: wstats.Timestamp.UnixNano(),
-				WorkingSetBytes: &runtime.UInt64Value{
-					Value: wstats.Memory.MemoryUsagePrivateWorkingSetBytes,
-				},
-			}
-		}
-	}
-	return &cs, nil
-}

pkg/cri/sbserver/sandbox_stats_windows.go

@@ -36,7 +36,8 @@ import (
 
 func (c *criService) podSandboxStats(
 	ctx context.Context,
-	sandbox sandboxstore.Sandbox) (*runtime.PodSandboxStats, error) {
+	sandbox sandboxstore.Sandbox,
+) (*runtime.PodSandboxStats, error) {
 	meta := sandbox.Metadata
 
 	if sandbox.Status.Get().State != sandboxstore.StateReady {