github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add code to fix kubelet/metrics memory issue.

Browse Source 
Bucketing url paths based on concept/handling.
Bucketing code placed by handling code to encourage usage.
Added unit tests.
Fix format.
This commit is contained in:
Walter Fender
2020-02-06 19:10:18 -08:00
parent 6a92f19444
commit 9802bfcec0
2 changed files with 105 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
pkg/kubelet/server/server.go
View File

@@ -89,6 +89,7 @@ type Server struct {
auth AuthInterface
host HostInterface
restfulCont containerInterface
metricsBuckets map[string]bool
resourceAnalyzer stats.ResourceAnalyzer
redirectContainerStreaming bool
}
@@ -225,6 +226,7 @@ func NewServer(
resourceAnalyzer: resourceAnalyzer,
auth: auth,
restfulCont: &filteringContainer{Container: restful.NewContainer()},
metricsBuckets: make(map[string]bool),
redirectContainerStreaming: redirectContainerStreaming,
}
if auth != nil {
@@ -280,14 +282,32 @@ func (s *Server) InstallAuthFilter() {
})
}
// addMetricsBucketMatcher adds a regexp matcher and the relevant bucket to use when
// it matches. Please be aware this is not thread safe and should not be used dynamically
func (s *Server) addMetricsBucketMatcher(bucket string) {
s.metricsBuckets[bucket] = true
}
// getMetricBucket find the appropriate metrics reporting bucket for the given path
func (s *Server) getMetricBucket(path string) string {
root := getURLRootPath(path)
if s.metricsBuckets[root] == true {
return root
}
return "Invalid path"
}
// InstallDefaultHandlers registers the default set of supported HTTP request
// patterns with the restful Container.
func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
s.addMetricsBucketMatcher("healthz")
healthz.InstallHandler(s.restfulCont,
healthz.PingHealthz,
healthz.LogHealthz,
healthz.NamedCheck("syncloop", s.syncLoopHealthCheck),
)
s.addMetricsBucketMatcher("pods")
ws := new(restful.WebService)
ws.
Path("/pods").
@@ -297,7 +317,14 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
Operation("getPods"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("stats")
s.restfulCont.Add(stats.CreateHandlers(statsPath, s.host, s.resourceAnalyzer, enableCAdvisorJSONEndpoints))
s.addMetricsBucketMatcher("metrics")
s.addMetricsBucketMatcher("metrics/cadvisor")
s.addMetricsBucketMatcher("metrics/probes")
s.addMetricsBucketMatcher("metrics/resource/v1alpha1")
s.addMetricsBucketMatcher("metrics/resource")
//lint:ignore SA1019 https://github.com/kubernetes/enhancements/issues/1206
s.restfulCont.Handle(metricsPath, legacyregistry.Handler())
@@ -321,12 +348,14 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
)
// deprecated endpoint which will be removed in release 1.20.0+.
s.addMetricsBucketMatcher("metrics/resource/v1alpha1")
v1alpha1ResourceRegistry := compbasemetrics.NewKubeRegistry()
v1alpha1ResourceRegistry.CustomMustRegister(stats.NewPrometheusResourceMetricCollector(s.resourceAnalyzer, v1alpha1.Config()))
s.restfulCont.Handle(path.Join(resourceMetricsPath, v1alpha1.Version),
compbasemetrics.HandlerFor(v1alpha1ResourceRegistry, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}),
)
s.addMetricsBucketMatcher("metrics/resource")
resourceRegistry := compbasemetrics.NewKubeRegistry()
resourceRegistry.CustomMustRegister(collectors.NewResourceMetricsCollector(s.resourceAnalyzer))
s.restfulCont.Handle(resourceMetricsPath,
@@ -335,6 +364,7 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
// prober metrics are exposed under a different endpoint
s.addMetricsBucketMatcher("metrics/probes")
p := compbasemetrics.NewKubeRegistry()
_ = compbasemetrics.RegisterProcessStartTime(p.Register)
p.MustRegister(prober.ProberResults)
@@ -342,6 +372,7 @@ func (s *Server) InstallDefaultHandlers(enableCAdvisorJSONEndpoints bool) {
compbasemetrics.HandlerFor(p, compbasemetrics.HandlerOpts{ErrorHandling: compbasemetrics.ContinueOnError}),
)
s.addMetricsBucketMatcher("spec")
if enableCAdvisorJSONEndpoints {
ws := new(restful.WebService)
ws.
@@ -361,6 +392,7 @@ const pprofBasePath = "/debug/pprof/"
func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
klog.Infof("Adding debug handlers to kubelet server.")
s.addMetricsBucketMatcher("run")
ws := new(restful.WebService)
ws.
Path("/run")
@@ -372,6 +404,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getRun"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("exec")
ws = new(restful.WebService)
ws.
Path("/exec")
@@ -389,6 +422,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getExec"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("attach")
ws = new(restful.WebService)
ws.
Path("/attach")
@@ -406,6 +440,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getAttach"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("portForward")
ws = new(restful.WebService)
ws.
Path("/portForward")
@@ -423,6 +458,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getPortForward"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("logs")
ws = new(restful.WebService)
ws.
Path(logsPath)
@@ -435,6 +471,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Param(ws.PathParameter("logpath", "path to the log").DataType("string")))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("containerLogs")
ws = new(restful.WebService)
ws.
Path("/containerLogs")
@@ -443,8 +480,10 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getContainerLogs"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("configz")
configz.InstallHandler(s.restfulCont)
s.addMetricsBucketMatcher("debug")
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
@@ -460,7 +499,6 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws = new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(func(req *restful.Request, resp *restful.Response) {
@@ -473,6 +511,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
s.restfulCont.Handle("/debug/flags/v", routes.StringFlagPutHandler(logs.GlogSetter))
// The /runningpods endpoint is used for testing only.
s.addMetricsBucketMatcher("runningpods")
ws = new(restful.WebService)
ws.
Path("/runningpods/").
@@ -482,6 +521,7 @@ func (s *Server) InstallDebuggingHandlers(criHandler http.Handler) {
Operation("getRunningPods"))
s.restfulCont.Add(ws)
s.addMetricsBucketMatcher("cri")
if criHandler != nil {
s.restfulCont.Handle("/cri/", criHandler)
}
@@ -493,6 +533,14 @@ func (s *Server) InstallDebuggingDisabledHandlers() {
http.Error(w, "Debug endpoints are disabled.", http.StatusMethodNotAllowed)
})
s.addMetricsBucketMatcher("run")
s.addMetricsBucketMatcher("exec")
s.addMetricsBucketMatcher("attach")
s.addMetricsBucketMatcher("portForward")
s.addMetricsBucketMatcher("containerLogs")
s.addMetricsBucketMatcher("runningpods")
s.addMetricsBucketMatcher("pprof")
s.addMetricsBucketMatcher("logs")
paths := []string{
"/run/", "/exec/", "/attach/", "/portForward/", "/containerLogs/",
"/runningpods/", pprofBasePath, logsPath}
@@ -809,10 +857,10 @@ func (s *Server) getPortForward(request *restful.Request, response *restful.Resp
proxyStream(response.ResponseWriter, request.Request, url)
}
// trimURLPath trims a URL path.
// getURLRootPath trims a URL path.
// For paths in the format of "/metrics/xxx", "metrics/xxx" is returned;
// For all other paths, the first part of the path is returned.
func trimURLPath(path string) string {
func getURLRootPath(path string) string {
parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3)
if len(parts) == 0 {
return path
@@ -860,7 +908,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
serverType = "readwrite"
}
method, path := req.Method, trimURLPath(req.URL.Path)
method, path := req.Method, s.getMetricBucket(req.URL.Path)
longRunning := strconv.FormatBool(isLongRunningRequest(path))