Merge pull request #11003 from jayunit100/prom-push

[contrib + e2e] prometheus: add pushgateway functionality to prometheus allinone deployment along w/ runRC Gauge reporting.

test/e2e/e2e_test.go

@@ -80,6 +80,7 @@ func init() {
 	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
 	flag.IntVar(&testContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
 	flag.StringVar(&testContext.UpgradeTarget, "upgrade-target", "latest_ci", "Version to upgrade to (e.g. 'latest_stable', 'latest_release', 'latest_ci', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
+	flag.StringVar(&testContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 }
 
 func TestE2E(t *testing.T) {

test/e2e/prompush.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//This is a utility for prometheus pushing functionality.
+package e2e
+
+import (
+	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Prometheus stuff: Setup metrics.
+var runningMetric = prometheus.NewGauge(prometheus.GaugeOpts{
+	Name: "e2e_running",
+	Help: "The num of running pods",
+})
+
+var pendingMetric = prometheus.NewGauge(prometheus.GaugeOpts{
+	Name: "e2e_pending",
+	Help: "The num of pending pods",
+})
+
+// Turn this to true after we register.
+var prom_registered = false
+
+// Reusable function for pushing metrics to prometheus.  Handles initialization and so on.
+func promPushRunningPending(running, pending int) error {
+
+	if testContext.PrometheusPushGateway == "" {
+		Logf("Ignoring prom push, push gateway unavailable")
+		return nil
+	} else {
+		// Register metrics if necessary
+		if !prom_registered && testContext.PrometheusPushGateway != "" {
+			prometheus.Register(runningMetric)
+			prometheus.Register(pendingMetric)
+			prom_registered = true
+		}
+		// Update metric values
+		runningMetric.Set(float64(running))
+		pendingMetric.Set(float64(pending))
+
+		// Push them to the push gateway.  This will be scraped by prometheus
+		// provided you launch it with the pushgateway as an endpoint.
+		if err := prometheus.Push(
+			"e2e",
+			"none",
+			testContext.PrometheusPushGateway, //i.e. "127.0.0.1:9091"
+		); err != nil {
+			fmt.Println("failed at pushing to pushgateway ", err)
+			return err
+		}
+	}
+	return nil
+}

test/e2e/util.go

@@ -105,18 +105,19 @@ type CloudConfig struct {
 }
 
 type TestContextType struct {
-	KubeConfig     string
-	KubeContext    string
-	CertDir        string
-	Host           string
-	RepoRoot       string
-	Provider       string
-	CloudConfig    CloudConfig
-	KubectlPath    string
-	OutputDir      string
-	prefix         string
-	MinStartupPods int
-	UpgradeTarget  string
+	KubeConfig            string
+	KubeContext           string
+	CertDir               string
+	Host                  string
+	RepoRoot              string
+	Provider              string
+	CloudConfig           CloudConfig
+	KubectlPath           string
+	OutputDir             string
+	prefix                string
+	MinStartupPods        int
+	UpgradeTarget         string
+	PrometheusPushGateway string
 }
 
 var testContext TestContextType
@@ -1150,6 +1151,9 @@ func RunRC(config RCConfig) error {
 
 		Logf("%v %v Pods: %d out of %d created, %d running, %d pending, %d waiting, %d inactive, %d unknown ",
 			time.Now(), rc.Name, len(pods), config.Replicas, running, pending, waiting, inactive, unknown)
+
+		promPushRunningPending(running, pending)
+
 		if config.PodStatusFile != nil {
 			fmt.Fprintf(config.PodStatusFile, "%s, %d, running, %d, pending, %d, waiting, %d, inactive, %d, unknown\n", time.Now(), running, pending, waiting, inactive, unknown)
 		}