7d180b337b
Relisting latency/interval affects how quick kubelet discovers changes. Record the metrics in Prometheus to surface such information.
141 lines
10 KiB
Go
141 lines
10 KiB
Go
/*
|
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"k8s.io/kubernetes/pkg/util/sets"
|
|
|
|
"github.com/prometheus/common/model"
|
|
)
|
|
|
|
var KnownKubeletMetrics = map[string][]string{
|
|
"cadvisor_version_info": {"cadvisorRevision", "cadvisorVersion", "dockerVersion", "kernelVersion", "osVersion"},
|
|
"container_cpu_system_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_cpu_usage_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "cpu"},
|
|
"container_cpu_user_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_current": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_time_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_time_weighted_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_limit_bytes": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_read_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_reads_merged_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_reads_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_sector_reads_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_sector_writes_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_usage_bytes": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_write_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_writes_merged_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_writes_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_last_seen": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_failcnt": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_failures_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "scope", "type"},
|
|
"container_memory_usage_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_working_set_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_bytes_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_errors_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_packets_dropped_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_packets_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_bytes_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_errors_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_packets_dropped_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_packets_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_scrape_error": {},
|
|
"container_spec_cpu_shares": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_spec_memory_limit_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_spec_memory_swap_limit_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_start_time_seconds": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_tasks_state": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "state"},
|
|
"kubelet_container_manager_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_container_manager_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_container_manager_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_containers_per_pod_count": {"quantile"},
|
|
"kubelet_containers_per_pod_count_count": {},
|
|
"kubelet_containers_per_pod_count_sum": {},
|
|
"kubelet_docker_errors": {"operation_type"},
|
|
"kubelet_docker_operations_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_docker_operations_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_docker_operations_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_generate_pod_status_latency_microseconds": {"quantile"},
|
|
"kubelet_generate_pod_status_latency_microseconds_count": {},
|
|
"kubelet_generate_pod_status_latency_microseconds_sum": {},
|
|
"kubelet_pleg_relist_latency_microseconds": {"quantile"},
|
|
"kubelet_pleg_relist_latency_microseconds_sum": {},
|
|
"kubelet_pleg_relist_latency_microseconds_count": {},
|
|
"kubelet_pleg_relist_interval_microseconds": {"quantile"},
|
|
"kubelet_pleg_relist_interval_microseconds_sum": {},
|
|
"kubelet_pleg_relist_interval_microseconds_count": {},
|
|
"kubelet_pod_start_latency_microseconds": {"quantile"},
|
|
"kubelet_pod_start_latency_microseconds_count": {},
|
|
"kubelet_pod_start_latency_microseconds_sum": {},
|
|
"kubelet_pod_worker_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_pod_worker_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_pod_worker_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_pod_worker_start_latency_microseconds": {"quantile"},
|
|
"kubelet_pod_worker_start_latency_microseconds_count": {},
|
|
"kubelet_pod_worker_start_latency_microseconds_sum": {},
|
|
"kubelet_running_container_count": {},
|
|
"kubelet_running_pod_count": {},
|
|
"kubelet_sync_pods_latency_microseconds": {"quantile"},
|
|
"kubelet_sync_pods_latency_microseconds_count": {},
|
|
"kubelet_sync_pods_latency_microseconds_sum": {},
|
|
"machine_cpu_cores": {},
|
|
"machine_memory_bytes": {},
|
|
"rest_client_request_latency_microseconds": {"quantile", "url", "verb"},
|
|
"rest_client_request_latency_microseconds_count": {"url", "verb"},
|
|
"rest_client_request_latency_microseconds_sum": {"url", "verb"},
|
|
"rest_client_request_status_codes": {"code", "host", "method"},
|
|
}
|
|
|
|
var KubeletMetricsLabelsToSkip = sets.NewString(
|
|
"kubernetes_namespace",
|
|
"image",
|
|
"name",
|
|
)
|
|
|
|
type KubeletMetrics Metrics
|
|
|
|
func NewKubeletMetrics() KubeletMetrics {
|
|
result := NewMetrics()
|
|
for metric := range KnownKubeletMetrics {
|
|
result[metric] = make(model.Samples, 0)
|
|
}
|
|
return KubeletMetrics(result)
|
|
}
|
|
|
|
func parseKubeletMetrics(data string, unknownMetrics sets.String) (KubeletMetrics, error) {
|
|
result := NewKubeletMetrics()
|
|
if err := parseMetrics(data, KnownKubeletMetrics, (*Metrics)(&result), unknownMetrics); err != nil {
|
|
return KubeletMetrics{}, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (g *MetricsGrabber) getMetricsFromNode(nodeName string, kubeletPort int) (string, error) {
|
|
rawOutput, err := g.client.Get().
|
|
Prefix("proxy").
|
|
Resource("nodes").
|
|
Name(fmt.Sprintf("%v:%v", nodeName, kubeletPort)).
|
|
Suffix("metrics").
|
|
Do().Raw()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(rawOutput), nil
|
|
}
|