e2e: wait for controller manager pod to be ready

The MetricsGrabber may use the controller-manager pod
to gather metrics, however, it doesn't wait until
it is ready to serve, failing the test if this is the
case.

We wait until the controller-manager pod is running
before trying to get metrics from it.

test/e2e/framework/metrics/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
         "//test/e2e/framework/log:go_default_library",
+        "//test/e2e/framework/pod:go_default_library",
         "//test/e2e/perftype:go_default_library",
         "//test/e2e/system:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",

test/e2e/framework/metrics/metrics_grabber.go

@@ -19,12 +19,14 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	clientset "k8s.io/client-go/kubernetes"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/master/ports"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/system"
 
 	"k8s.io/klog"
@@ -41,15 +43,16 @@ type Collection struct {
 
 // Grabber provides functions which grab metrics from components
 type Grabber struct {
-	client                    clientset.Interface
-	externalClient            clientset.Interface
-	grabFromAPIServer         bool
-	grabFromControllerManager bool
-	grabFromKubelets          bool
-	grabFromScheduler         bool
-	grabFromClusterAutoscaler bool
-	masterName                string
-	registeredMaster          bool
+	client                            clientset.Interface
+	externalClient                    clientset.Interface
+	grabFromAPIServer                 bool
+	grabFromControllerManager         bool
+	grabFromKubelets                  bool
+	grabFromScheduler                 bool
+	grabFromClusterAutoscaler         bool
+	masterName                        string
+	registeredMaster                  bool
+	waitForControllerManagerReadyOnce sync.Once
 }
 
 // NewMetricsGrabber returns new metrics which are initialized.
@@ -161,7 +164,12 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error)
 	if !g.registeredMaster {
 		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
 	}
-	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
+
+	podName := fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName)
+	g.waitForControllerManagerReadyOnce.Do(func() {
+		e2epod.WaitForPodNameRunningInNamespace(g.client, podName, metav1.NamespaceSystem)
+	})
+	output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
 	if err != nil {
 		return ControllerManagerMetrics{}, err
 	}