github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix volume metric flake

Browse Source 
Make sure we only run this test in environments
that support it.
This commit is contained in:
Hemant Kumar
2017-09-21 19:50:14 -04:00
committed by Hemant Kumar
parent 64fde98206
commit 381e334d87
2 changed files with 39 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
test/e2e/framework/metrics/metrics_grabber.go
View File

@@ -95,6 +95,11 @@ func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets b
}, nil
}
// HasRegisteredMaster returns if metrics grabber was able to find a master node
func (g *MetricsGrabber) HasRegisteredMaster() bool {
return g.registeredMaster
}
func (g *MetricsGrabber) GrabFromKubelet(nodeName string) (KubeletMetrics, error) {
nodes, err := g.client.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{api.ObjectNameField: nodeName}.AsSelector().String()})
if err != nil {

36
test/e2e/storage/volume_metrics.go
View File

@@ -25,6 +25,7 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/test/e2e/framework"
@@ -61,6 +62,10 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
if err != nil {
framework.Failf("Error creating metrics grabber : %v", err)
}
if !metricsGrabber.HasRegisteredMaster() {
framework.Skipf("Environment does not support getting controller-manager metrics - skipping")
}
})
AfterEach(func() {
@@ -71,6 +76,7 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
var err error
controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
Expect(err).NotTo(HaveOccurred(), "Error getting c-m metrics : %v", err)
storageOpMetrics := getControllerStorageMetrics(controllerMetrics)
@@ -91,9 +97,30 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
backoff := wait.Backoff{
Duration: 10 * time.Second,
Factor: 1.2,
Steps: 3,
}
updatedStorageMetrics := make(map[string]int64)
waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) {
updatedMetrics, err := metricsGrabber.GrabFromControllerManager()
Expect(err).NotTo(HaveOccurred(), "Error getting c-m metrics : %v", err)
updatedStorageMetrics := getControllerStorageMetrics(updatedMetrics)
if err != nil {
framework.Logf("Error fetching controller-manager metrics")
return false, err
}
updatedStorageMetrics = getControllerStorageMetrics(updatedMetrics)
if len(updatedStorageMetrics) == 0 {
framework.Logf("Volume metrics not collected yet, going to retry")
return false, nil
}
return true, nil
})
Expect(waitErr).NotTo(HaveOccurred(), "Error fetching storage c-m metrics : %v", waitErr)
volumeOperations := []string{"volume_provision", "volume_detach", "volume_attach"}
for _, volumeOp := range volumeOperations {
@@ -147,7 +174,10 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
func verifyMetricCount(oldMetrics map[string]int64, newMetrics map[string]int64, metricName string) {
oldCount, ok := oldMetrics[metricName]
Expect(ok).To(BeTrue(), "Error getting metrics for %s", metricName)
// if metric does not exist in oldMap, it probably hasn't been emitted yet.
if !ok {
oldCount = 0
}
newCount, ok := newMetrics[metricName]
Expect(ok).To(BeTrue(), "Error getting updated metrics for %s", metricName)