From c525aa5f859237fdbb83b551c1ea84b044a7cd70 Mon Sep 17 00:00:00 2001 From: Nguyen Phan Huy Date: Wed, 6 Apr 2022 16:35:54 +0800 Subject: [PATCH] Set timeout when collecting metrics from shim's Stat Signed-off-by: Nguyen Phan Huy --- metrics/cgroups/v1/metrics.go | 10 ++++++++-- metrics/cgroups/v2/metrics.go | 10 ++++++++-- metrics/metrics.go | 8 ++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/metrics/cgroups/v1/metrics.go b/metrics/cgroups/v1/metrics.go index f73b0608a..f484442dd 100644 --- a/metrics/cgroups/v1/metrics.go +++ b/metrics/cgroups/v1/metrics.go @@ -26,9 +26,11 @@ import ( "github.com/containerd/cgroups" "github.com/containerd/containerd/log" + cmetrics "github.com/containerd/containerd/metrics" "github.com/containerd/containerd/metrics/cgroups/common" v1 "github.com/containerd/containerd/metrics/types/v1" "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/pkg/timeout" "github.com/containerd/typeurl" "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" @@ -136,13 +138,17 @@ func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool if wg != nil { defer wg.Done() } + t := entry.task - ctx := namespaces.WithNamespace(context.Background(), t.Namespace()) - stats, err := t.Stats(ctx) + ctx, cancel := timeout.WithContext(context.Background(), cmetrics.ShimStatsRequestTimeout) + stats, err := t.Stats(namespaces.WithNamespace(ctx, t.Namespace())) + cancel() + if err != nil { log.L.WithError(err).Errorf("stat task %s", t.ID()) return } + data, err := typeurl.UnmarshalAny(stats) if err != nil { log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID()) diff --git a/metrics/cgroups/v2/metrics.go b/metrics/cgroups/v2/metrics.go index 00c498c83..b34cc4a82 100644 --- a/metrics/cgroups/v2/metrics.go +++ b/metrics/cgroups/v2/metrics.go @@ -25,9 +25,11 @@ import ( "sync" "github.com/containerd/containerd/log" + cmetrics "github.com/containerd/containerd/metrics" "github.com/containerd/containerd/metrics/cgroups/common" v2 "github.com/containerd/containerd/metrics/types/v2" "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/pkg/timeout" "github.com/containerd/typeurl" "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" @@ -129,13 +131,17 @@ func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool if wg != nil { defer wg.Done() } + t := entry.task - ctx := namespaces.WithNamespace(context.Background(), t.Namespace()) - stats, err := t.Stats(ctx) + ctx, cancel := timeout.WithContext(context.Background(), cmetrics.ShimStatsRequestTimeout) + stats, err := t.Stats(namespaces.WithNamespace(ctx, t.Namespace())) + cancel() + if err != nil { log.L.WithError(err).Errorf("stat task %s", t.ID()) return } + data, err := typeurl.UnmarshalAny(stats) if err != nil { log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID()) diff --git a/metrics/metrics.go b/metrics/metrics.go index 1debd87dc..86ee4d59b 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -17,13 +17,21 @@ package metrics import ( + "time" + + "github.com/containerd/containerd/pkg/timeout" "github.com/containerd/containerd/version" goMetrics "github.com/docker/go-metrics" ) +const ( + ShimStatsRequestTimeout = "io.containerd.timeout.metrics.shimstats" +) + func init() { ns := goMetrics.NewNamespace("containerd", "", nil) c := ns.NewLabeledCounter("build_info", "containerd build information", "version", "revision") c.WithValues(version.Version, version.Revision).Inc() goMetrics.Register(ns) + timeout.Set(ShimStatsRequestTimeout, 2*time.Second) }