Add e2e test for Custom Metrics API with new Stackdriver resource model
and External Metrics API.
This commit is contained in:
Karol Wychowaniec
@@ -34,6 +34,7 @@ import (
|
||||
"k8s.io/client-go/discovery"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
customclient "k8s.io/metrics/pkg/client/custom_metrics"
|
||||
externalclient "k8s.io/metrics/pkg/client/external_metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -48,41 +49,46 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() {
|
||||
})
|
||||
|
||||
f := framework.NewDefaultFramework("stackdriver-monitoring")
|
||||
var kubeClient clientset.Interface
|
||||
var customMetricsClient customclient.CustomMetricsClient
|
||||
var discoveryClient *discovery.DiscoveryClient
|
||||
|
||||
It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() {
|
||||
kubeClient = f.ClientSet
|
||||
It("should run Custom Metrics - Stackdriver Adapter for old resource model [Feature:StackdriverCustomMetrics]", func() {
|
||||
kubeClient := f.ClientSet
|
||||
config, err := framework.LoadConfig()
|
||||
if err != nil {
|
||||
framework.Failf("Failed to load config: %s", err)
|
||||
}
|
||||
customMetricsClient = customclient.NewForConfigOrDie(config)
|
||||
discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config)
|
||||
testAdapter(f, kubeClient, customMetricsClient, discoveryClient)
|
||||
customMetricsClient := customclient.NewForConfigOrDie(config)
|
||||
discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
|
||||
testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForOldResourceModel)
|
||||
})
|
||||
|
||||
It("should run Custom Metrics - Stackdriver Adapter for new resource model [Feature:StackdriverCustomMetrics]", func() {
|
||||
kubeClient := f.ClientSet
|
||||
config, err := framework.LoadConfig()
|
||||
if err != nil {
|
||||
framework.Failf("Failed to load config: %s", err)
|
||||
}
|
||||
customMetricsClient := customclient.NewForConfigOrDie(config)
|
||||
discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
|
||||
testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForNewResourceModel)
|
||||
})
|
||||
|
||||
It("should run Custom Metrics - Stackdriver Adapter for external metrics [Feature:StackdriverExternalMetrics]", func() {
|
||||
kubeClient := f.ClientSet
|
||||
config, err := framework.LoadConfig()
|
||||
if err != nil {
|
||||
framework.Failf("Failed to load config: %s", err)
|
||||
}
|
||||
externalMetricsClient := externalclient.NewForConfigOrDie(config)
|
||||
testExternalMetrics(f, kubeClient, externalMetricsClient)
|
||||
})
|
||||
})
|
||||
|
||||
func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
|
||||
func testCustomMetrics(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient, adapterDeployment string) {
|
||||
projectId := framework.TestContext.CloudConfig.ProjectID
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
|
||||
|
||||
// Hack for running tests locally, needed to authenticate in Stackdriver
|
||||
// If this is your use case, create application default credentials:
|
||||
// $ gcloud auth application-default login
|
||||
// and uncomment following lines (comment out the two lines above):
|
||||
/*
|
||||
ts, err := google.DefaultTokenSource(oauth2.NoContext)
|
||||
framework.Logf("Couldn't get application default credentials, %v", err)
|
||||
if err != nil {
|
||||
framework.Failf("Error accessing application default credentials, %v", err)
|
||||
}
|
||||
client := oauth2.NewClient(oauth2.NoContext, ts)
|
||||
*/
|
||||
|
||||
gcmService, err := gcm.New(client)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to create gcm service, %v", err)
|
||||
@@ -95,11 +101,11 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
|
||||
}
|
||||
defer CleanupDescriptors(gcmService, projectId)
|
||||
|
||||
err = CreateAdapter()
|
||||
err = CreateAdapter(adapterDeployment)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to set up: %s", err)
|
||||
}
|
||||
defer CleanupAdapter()
|
||||
defer CleanupAdapter(adapterDeployment)
|
||||
|
||||
_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
|
||||
defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
|
||||
@@ -116,16 +122,62 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
|
||||
// i.e. pod creation, first time series exported
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
// Verify responses from Custom Metrics API
|
||||
verifyResponsesFromCustomMetricsAPI(f, customMetricsClient, discoveryClient)
|
||||
}
|
||||
|
||||
func testExternalMetrics(f *framework.Framework, kubeClient clientset.Interface, externalMetricsClient externalclient.ExternalMetricsClient) {
|
||||
projectId := framework.TestContext.CloudConfig.ProjectID
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
|
||||
|
||||
gcmService, err := gcm.New(client)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to create gcm service, %v", err)
|
||||
}
|
||||
|
||||
// Set up a cluster: create a custom metric and set up k8s-sd adapter
|
||||
err = CreateDescriptors(gcmService, projectId)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to create metric descriptor: %s", err)
|
||||
}
|
||||
defer CleanupDescriptors(gcmService, projectId)
|
||||
|
||||
// Both deployments - for old and new resource model - expose External Metrics API.
|
||||
err = CreateAdapter(AdapterForNewResourceModel)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to set up: %s", err)
|
||||
}
|
||||
defer CleanupAdapter(AdapterForNewResourceModel)
|
||||
|
||||
_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
|
||||
defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
|
||||
|
||||
// Run application that exports the metric
|
||||
err = createSDExporterPods(f, kubeClient)
|
||||
if err != nil {
|
||||
framework.Failf("Failed to create stackdriver-exporter pod: %s", err)
|
||||
}
|
||||
defer cleanupSDExporterPod(f, kubeClient)
|
||||
|
||||
// Wait a short amount of time to create a pod and export some metrics
|
||||
// TODO: add some events to wait for instead of fixed amount of time
|
||||
// i.e. pod creation, first time series exported
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
verifyResponseFromExternalMetricsAPI(f, externalMetricsClient)
|
||||
}
|
||||
|
||||
func verifyResponsesFromCustomMetricsAPI(f *framework.Framework, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
|
||||
resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1")
|
||||
if err != nil {
|
||||
framework.Failf("Failed to retrieve a list of supported metrics: %s", err)
|
||||
}
|
||||
gotCustomMetric, gotUnusedMetric := false, false
|
||||
for _, resource := range resources.APIResources {
|
||||
if resource.Name == "pods/"+CustomMetricName {
|
||||
if resource.Name == "*/"+CustomMetricName {
|
||||
gotCustomMetric = true
|
||||
} else if resource.Name == "pods/"+UnusedMetricName {
|
||||
} else if resource.Name == "*/"+UnusedMetricName {
|
||||
gotUnusedMetric = true
|
||||
} else {
|
||||
framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName)
|
||||
@@ -160,6 +212,31 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
|
||||
}
|
||||
}
|
||||
|
||||
func verifyResponseFromExternalMetricsAPI(f *framework.Framework, externalMetricsClient externalclient.ExternalMetricsClient) {
|
||||
req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"k8s_pod"})
|
||||
// It's important to filter out only metrics from the right namespace, since multiple e2e tests
|
||||
// may run in the same project concurrently. "dummy" is added to test
|
||||
req2, _ := labels.NewRequirement("resource.label.namespace_name", selection.In, []string{string(f.Namespace.Name), "dummy"})
|
||||
req3, _ := labels.NewRequirement("resource.label.pod_name", selection.Exists, []string{})
|
||||
req4, _ := labels.NewRequirement("resource.label.location", selection.NotEquals, []string{"dummy"})
|
||||
req5, _ := labels.NewRequirement("resource.label.cluster_name", selection.NotIn, []string{"foo", "bar"})
|
||||
values, err := externalMetricsClient.
|
||||
NamespacedMetrics("dummy").
|
||||
List("custom.googleapis.com|"+CustomMetricName, labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5))
|
||||
if err != nil {
|
||||
framework.Failf("Failed query: %s", err)
|
||||
}
|
||||
if len(values.Items) != 1 {
|
||||
framework.Failf("Expected exactly one external metric value, but % values received", len(values.Items))
|
||||
}
|
||||
if values.Items[0].MetricName != "custom.googleapis.com|"+CustomMetricName ||
|
||||
values.Items[0].Value.Value() != CustomMetricValue ||
|
||||
// Check one label just to make sure labels are included
|
||||
values.Items[0].MetricLabels["resource.label.namespace_name"] != string(f.Namespace.Name) {
|
||||
framework.Failf("Unexpected result for metric %s: %v", CustomMetricName, values.Items[0])
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) {
|
||||
err := cs.CoreV1().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user