Merge pull request #6781 from phanhuy1502/metrics-collector-stats-timeout

Set timeout when collecting metrics from shim's Stat

metrics/cgroups/v1/metrics.go

@@ -26,9 +26,11 @@ import (
 
 	"github.com/containerd/cgroups"
 	"github.com/containerd/containerd/log"
+	cmetrics "github.com/containerd/containerd/metrics"
 	"github.com/containerd/containerd/metrics/cgroups/common"
 	v1 "github.com/containerd/containerd/metrics/types/v1"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/pkg/timeout"
 	"github.com/containerd/typeurl"
 	"github.com/docker/go-metrics"
 	"github.com/prometheus/client_golang/prometheus"
@@ -136,13 +138,17 @@ func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool
 	if wg != nil {
 		defer wg.Done()
 	}
+
 	t := entry.task
-	ctx := namespaces.WithNamespace(context.Background(), t.Namespace())
+	ctx, cancel := timeout.WithContext(context.Background(), cmetrics.ShimStatsRequestTimeout)
-	stats, err := t.Stats(ctx)
+	stats, err := t.Stats(namespaces.WithNamespace(ctx, t.Namespace()))
+	cancel()
+
 	if err != nil {
 		log.L.WithError(err).Errorf("stat task %s", t.ID())
 		return
 	}
+
 	data, err := typeurl.UnmarshalAny(stats)
 	if err != nil {
 		log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID())

metrics/cgroups/v2/metrics.go

@@ -25,9 +25,11 @@ import (
 	"sync"
 
 	"github.com/containerd/containerd/log"
+	cmetrics "github.com/containerd/containerd/metrics"
 	"github.com/containerd/containerd/metrics/cgroups/common"
 	v2 "github.com/containerd/containerd/metrics/types/v2"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/pkg/timeout"
 	"github.com/containerd/typeurl"
 	"github.com/docker/go-metrics"
 	"github.com/prometheus/client_golang/prometheus"
@@ -129,13 +131,17 @@ func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool
 	if wg != nil {
 		defer wg.Done()
 	}
+
 	t := entry.task
-	ctx := namespaces.WithNamespace(context.Background(), t.Namespace())
+	ctx, cancel := timeout.WithContext(context.Background(), cmetrics.ShimStatsRequestTimeout)
-	stats, err := t.Stats(ctx)
+	stats, err := t.Stats(namespaces.WithNamespace(ctx, t.Namespace()))
+	cancel()
+
 	if err != nil {
 		log.L.WithError(err).Errorf("stat task %s", t.ID())
 		return
 	}
+
 	data, err := typeurl.UnmarshalAny(stats)
 	if err != nil {
 		log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID())

metrics/metrics.go

@@ -17,13 +17,21 @@
 package metrics
 
 import (
+	"time"
+
+	"github.com/containerd/containerd/pkg/timeout"
 	"github.com/containerd/containerd/version"
 	goMetrics "github.com/docker/go-metrics"
 )
 
+const (
+	ShimStatsRequestTimeout = "io.containerd.timeout.metrics.shimstats"
+)
+
 func init() {
 	ns := goMetrics.NewNamespace("containerd", "", nil)
 	c := ns.NewLabeledCounter("build_info", "containerd build information", "version", "revision")
 	c.WithValues(version.Version, version.Revision).Inc()
 	goMetrics.Register(ns)
+	timeout.Set(ShimStatsRequestTimeout, 2*time.Second)
 }