github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #95562 from wojtek-t/fix_metrics

Browse Source 
Fix metrics reporting in kube-apiserver
This commit is contained in:
Kubernetes Prow Robot
2020-10-15 21:13:25 -07:00
committed by GitHub
parent a5adff2b89 3d2a80643a
commit cd8b87f25a
4 changed files with 136 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build/visible_to/BUILD
View File

@@ -436,6 +436,7 @@ package_group(
"//staging/src/k8s.io/component-base/metrics/...",
"//test/e2e_node",
"//test/integration/apiserver/flowcontrol",
"//test/integration/metrics",
"//vendor/...",
],
)

3
staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
View File

@@ -594,6 +594,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
requestScope = "resource"
operationSuffix = operationSuffix + "WithPath"
}
if strings.Index(action.Path, "/{name}") != -1 || action.Verb == "POST" {
requestScope = "resource"
}
if action.AllNamespaces {
requestScope = "cluster"
operationSuffix = operationSuffix + "ForAllNamespaces"

2
test/integration/metrics/BUILD
View File

@@ -35,11 +35,13 @@ go_test(
embed = [":go_default_library"],
tags = ["integration"],
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
"//test/integration/framework:go_default_library",
"//vendor/github.com/prometheus/common/model:go_default_library",
],
)

130
test/integration/metrics/metrics_test.go
View File

@@ -25,6 +25,9 @@ import (
"runtime"
"testing"
"github.com/prometheus/common/model"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
clientset "k8s.io/client-go/kubernetes"
@@ -111,3 +114,130 @@ func TestApiserverMetrics(t *testing.T) {
"etcd_request_duration_seconds_sum",
})
}
func TestApiserverMetricsLabels(t *testing.T) {
_, s, closeFn := framework.RunAMaster(nil)
defer closeFn()
client, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1})
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
expectedMetrics := []model.Metric{}
metricLabels := func(group, version, resource, subresource, scope, verb string) model.Metric {
return map[model.LabelName]model.LabelValue{
model.LabelName("group"): model.LabelValue(group),
model.LabelName("version"): model.LabelValue(version),
model.LabelName("resource"): model.LabelValue(resource),
model.LabelName("subresource"): model.LabelValue(subresource),
model.LabelName("scope"): model.LabelValue(scope),
model.LabelName("verb"): model.LabelValue(verb),
}
}
callOrDie := func(_ interface{}, err error) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
appendExpectedMetric := func(metric model.Metric) {
expectedMetrics = append(expectedMetrics, metric)
}
// Call appropriate endpoints to ensure particular metrics will be exposed
// Namespace-scoped resource
c := client.CoreV1().Pods(metav1.NamespaceDefault)
makePod := func(labelValue string) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Labels: map[string]string{"foo": labelValue},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "container",
Image: "image",
},
},
},
}
}
callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "POST"))
callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "PUT"))
callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "status", "resource", "PUT"))
callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "GET"))
callOrDie(c.List(context.TODO(), metav1.ListOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "namespace", "LIST"))
callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "resource", "DELETE"))
// cluster-scoped LIST of namespace-scoped resources
callOrDie(client.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}))
appendExpectedMetric(metricLabels("", "v1", "pods", "", "cluster", "LIST"))
// Cluster-scoped resource
cn := client.CoreV1().Namespaces()
makeNamespace := func(labelValue string) *v1.Namespace {
return &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Labels: map[string]string{"foo": labelValue},
},
}
}
callOrDie(cn.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "POST"))
callOrDie(cn.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "PUT"))
callOrDie(cn.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "status", "resource", "PUT"))
callOrDie(cn.Get(context.TODO(), "foo", metav1.GetOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "GET"))
callOrDie(cn.List(context.TODO(), metav1.ListOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "cluster", "LIST"))
callOrDie(nil, cn.Delete(context.TODO(), "foo", metav1.DeleteOptions{}))
appendExpectedMetric(metricLabels("", "v1", "namespaces", "", "resource", "DELETE"))
// Verify if all metrics were properly exported.
metrics, err := scrapeMetrics(s)
if err != nil {
t.Fatal(err)
}
samples, ok := metrics["apiserver_request_total"]
if !ok {
t.Fatalf("apiserver_request_total metric not exposed")
}
hasLabels := func(current, expected model.Metric) bool {
for key, value := range expected {
if current[key] != value {
return false
}
}
return true
}
for _, expectedMetric := range expectedMetrics {
found := false
for _, sample := range samples {
if hasLabels(sample.Metric, expectedMetric) {
found = true
break
}
}
if !found {
t.Errorf("No sample found for %#v", expectedMetric)
}
}
}