From 72bcdb8fa93e2f1f3089b2ec75684bfb82c1b1c7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 7 Sep 2017 13:18:00 -0400 Subject: [PATCH] Add config for exporting container metrics to prom This adds an option for the cgroups monitor to include container metrics in the prometheus output. We will have to use the plugin to emit oom events via the events service but when the `no_prom` setting is set for the plugin container metrics will not be included in the prom output. Signed-off-by: Michael Crosby --- metrics/cgroups/blkio.go | 13 +++++++++++++ metrics/cgroups/cgroups.go | 25 +++++++++++++++++-------- metrics/cgroups/metrics.go | 33 +++++++++------------------------ metrics/cgroups/oom.go | 10 ++++++++-- 4 files changed, 47 insertions(+), 34 deletions(-) diff --git a/metrics/cgroups/blkio.go b/metrics/cgroups/blkio.go index f47be6c91..a310d487a 100644 --- a/metrics/cgroups/blkio.go +++ b/metrics/cgroups/blkio.go @@ -3,6 +3,8 @@ package cgroups import ( + "strconv" + "github.com/containerd/cgroups" metrics "github.com/docker/go-metrics" "github.com/prometheus/client_golang/prometheus" @@ -101,3 +103,14 @@ var blkioMetrics = []*metric{ }, }, } + +func blkioValues(l []*cgroups.BlkIOEntry) []value { + var out []value + for _, e := range l { + out = append(out, value{ + v: float64(e.Value), + l: []string{e.Op, e.Device, strconv.FormatUint(e.Major, 10), strconv.FormatUint(e.Minor, 10)}, + }) + } + return out +} diff --git a/metrics/cgroups/cgroups.go b/metrics/cgroups/cgroups.go index cab67ebbf..58d699f8f 100644 --- a/metrics/cgroups/cgroups.go +++ b/metrics/cgroups/cgroups.go @@ -15,24 +15,33 @@ import ( "golang.org/x/net/context" ) +type Config struct { + NoPrometheus bool `toml:"no_prometheus"` +} + func init() { plugin.Register(&plugin.Registration{ - Type: plugin.TaskMonitorPlugin, - ID: "cgroups", - Init: New, + Type: plugin.TaskMonitorPlugin, + ID: "cgroups", + Init: New, + Config: &Config{}, }) } func New(ic *plugin.InitContext) (interface{}, error) { - var ( - ns = metrics.NewNamespace("container", "", nil) - collector = NewCollector(ns) - ) + var ns *metrics.Namespace + config := ic.Config.(*Config) + if !config.NoPrometheus { + ns = metrics.NewNamespace("container", "", nil) + } + collector := NewCollector(ns) oom, err := NewOOMCollector(ns) if err != nil { return nil, err } - metrics.Register(ns) + if ns != nil { + metrics.Register(ns) + } return &cgroupsMonitor{ collector: collector, oom: oom, diff --git a/metrics/cgroups/metrics.go b/metrics/cgroups/metrics.go index 3039b8d58..9ef6bded8 100644 --- a/metrics/cgroups/metrics.go +++ b/metrics/cgroups/metrics.go @@ -5,7 +5,6 @@ package cgroups import ( "errors" "fmt" - "strconv" "sync" "github.com/containerd/cgroups" @@ -26,6 +25,9 @@ type Trigger func(string, string, cgroups.Cgroup) // New registers the Collector with the provided namespace and returns it so // that cgroups can be added for collection func NewCollector(ns *metrics.Namespace) *Collector { + if ns == nil { + return &Collector{} + } // add machine cpus and memory info c := &Collector{ ns: ns, @@ -91,6 +93,9 @@ func (c *Collector) collect(id, namespace string, cg cgroups.Cgroup, ch chan<- p // Add adds the provided cgroup and id so that metrics are collected and exported func (c *Collector) Add(id, namespace string, cg cgroups.Cgroup) error { + if c.ns == nil { + return nil + } c.mu.Lock() defer c.mu.Unlock() if _, ok := c.cgroups[taskID(id, namespace)]; ok { @@ -104,32 +109,12 @@ func (c *Collector) Add(id, namespace string, cg cgroups.Cgroup) error { return nil } -// Get returns the cgroup that is being collected under the provided id -// returns ErrCgroupNotExists if the id is not being collected -func (c *Collector) Get(id, namespace string) (cgroups.Cgroup, error) { - c.mu.Lock() - defer c.mu.Unlock() - t, ok := c.cgroups[taskID(id, namespace)] - if !ok { - return nil, ErrCgroupNotExists - } - return t.cgroup, nil -} - // Remove removes the provided cgroup by id from the collector func (c *Collector) Remove(id, namespace string) { + if c.ns == nil { + return + } c.mu.Lock() defer c.mu.Unlock() delete(c.cgroups, taskID(id, namespace)) } - -func blkioValues(l []*cgroups.BlkIOEntry) []value { - var out []value - for _, e := range l { - out = append(out, value{ - v: float64(e.Value), - l: []string{e.Op, e.Device, strconv.FormatUint(e.Major, 10), strconv.FormatUint(e.Minor, 10)}, - }) - } - return out -} diff --git a/metrics/cgroups/oom.go b/metrics/cgroups/oom.go index 39989d52d..c6b80ec2c 100644 --- a/metrics/cgroups/oom.go +++ b/metrics/cgroups/oom.go @@ -19,12 +19,18 @@ func NewOOMCollector(ns *metrics.Namespace) (*OOMCollector, error) { if err != nil { return nil, err } + var desc *prometheus.Desc + if ns != nil { + desc = ns.NewDesc("memory_oom", "The number of times a container has received an oom event", metrics.Total, "container_id", "namespace") + } c := &OOMCollector{ fd: fd, - desc: ns.NewDesc("memory_oom", "The number of times a container has received an oom event", metrics.Total, "container_id", "namespace"), + desc: desc, set: make(map[uintptr]*oom), } - ns.Add(c) + if ns != nil { + ns.Add(c) + } go c.start() return c, nil }