diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go index d38cbb0c3cd..55e1ce5aa8a 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go @@ -37,6 +37,7 @@ import ( "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/finisher" + requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" @@ -86,7 +87,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int return } - body, err := limitedReadBody(req, scope.MaxRequestBodyBytes) + body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Create) trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err}) if err != nil { scope.err(err, w, req) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go index dab250b0827..6bbf8c19752 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go @@ -33,6 +33,7 @@ import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/endpoints/handlers/finisher" + requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" @@ -70,7 +71,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc options := &metav1.DeleteOptions{} if allowsOptions { - body, err := limitedReadBody(req, scope.MaxRequestBodyBytes) + body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Patch) if err != nil { scope.err(err, w, req) return diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/OWNERS b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/OWNERS new file mode 100644 index 00000000000..433e84aa3e4 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/OWNERS @@ -0,0 +1,4 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - logicalhan diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/metrics.go new file mode 100644 index 00000000000..a870f3db418 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/metrics/metrics.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "k8s.io/component-base/metrics" +) + +type RequestBodyVerb string + +const ( + Patch RequestBodyVerb = "patch" + Delete RequestBodyVerb = "delete" + Update RequestBodyVerb = "update" + Create RequestBodyVerb = "create" +) + +var ( + RequestBodySizes = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: "apiserver", + Name: "request_body_sizes", + Help: "Apiserver request body sizes broken out by size.", + // we use 0.05 KB as the smallest bucket with 0.1 KB increments up to the + // apiserver limit. + Buckets: metrics.LinearBuckets(50000, 100000, 31), + StabilityLevel: metrics.ALPHA, + }, + []string{"resource", "verb"}, + ) +) + +func RecordRequestBodySize(ctx context.Context, resource string, verb RequestBodyVerb, size int) { + RequestBodySizes.WithContext(ctx).WithLabelValues(resource, string(verb)).Observe(float64(size)) +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go index 98909533edb..1800b8f5b0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go @@ -23,9 +23,9 @@ import ( "strings" "time" + jsonpatch "github.com/evanphx/json-patch" kjson "sigs.k8s.io/json" - jsonpatch "github.com/evanphx/json-patch" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" @@ -45,6 +45,7 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/finisher" + requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" @@ -98,7 +99,7 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac return } - patchBytes, err := limitedReadBody(req, scope.MaxRequestBodyBytes) + patchBytes, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Patch) trace.Step("limitedReadBody done", utiltrace.Field{"len", len(patchBytes)}, utiltrace.Field{"err", err}) if err != nil { scope.err(err, w, req) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go index e413661e760..7f005a37167 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go @@ -41,6 +41,7 @@ import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" + requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/apiserver/pkg/endpoints/request" @@ -390,6 +391,15 @@ func limitedReadBody(req *http.Request, limit int64) ([]byte, error) { return data, nil } +func limitedReadBodyWithRecordMetric(ctx context.Context, req *http.Request, limit int64, resourceGroup string, verb requestmetrics.RequestBodyVerb) ([]byte, error) { + readBody, err := limitedReadBody(req, limit) + if err == nil { + // only record if we've read successfully + requestmetrics.RecordRequestBodySize(ctx, resourceGroup, verb, len(readBody)) + } + return readBody, err +} + func isDryRun(url *url.URL) bool { return len(url.Query()["dryRun"]) != 0 } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go index 52d169a86e4..5518839523b 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "io" "net/http" "reflect" "strings" @@ -45,9 +46,12 @@ import ( "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/apis/example" examplev1 "k8s.io/apiserver/pkg/apis/example/v1" + "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" utiltrace "k8s.io/utils/trace" ) @@ -107,6 +111,94 @@ func TestPatchAnonymousField(t *testing.T) { } } +func TestLimitedReadBody(t *testing.T) { + defer legacyregistry.Reset() + legacyregistry.Register(metrics.RequestBodySizes) + + testcases := []struct { + desc string + requestBody io.Reader + limit int64 + expectedMetrics string + expectedErr bool + }{ + { + desc: "aaaa with limit 1", + requestBody: strings.NewReader("aaaa"), + limit: 1, + expectedMetrics: "", + expectedErr: true, + }, + { + desc: "aaaa with limit 5", + requestBody: strings.NewReader("aaaa"), + limit: 5, + expectedMetrics: ` + # HELP apiserver_request_body_sizes [ALPHA] Apiserver request body sizes broken out by size. + # TYPE apiserver_request_body_sizes histogram + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="50000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="150000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="250000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="350000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="450000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="550000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="650000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="750000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="850000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="950000"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.05e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.15e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.25e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.35e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.45e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.55e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.65e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.75e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.85e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="1.95e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.05e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.15e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.25e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.35e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.45e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.55e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.65e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.75e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.85e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="2.95e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="3.05e+06"} 1 + apiserver_request_body_sizes_bucket{resource="resource.group",verb="create",le="+Inf"} 1 + apiserver_request_body_sizes_sum{resource="resource.group",verb="create"} 4 + apiserver_request_body_sizes_count{resource="resource.group",verb="create"} 1 +`, + expectedErr: false, + }, + } + + for _, tc := range testcases { + t.Run(tc.desc, func(t *testing.T) { + // reset metrics + defer metrics.RequestBodySizes.Reset() + defer legacyregistry.Reset() + + req, err := http.NewRequest("POST", "/", tc.requestBody) + if err != nil { + t.Errorf("err not expected: got %v", err) + } + _, err = limitedReadBodyWithRecordMetric(context.Background(), req, tc.limit, "resource.group", metrics.Create) + if tc.expectedErr { + if err == nil { + t.Errorf("err expected: got nil") + } + return + } + if err = testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedMetrics), "apiserver_request_body_sizes"); err != nil { + t.Errorf("unexpected err: %v", err) + } + }) + } +} + func TestStrategicMergePatchInvalid(t *testing.T) { testGV := schema.GroupVersion{Group: "", Version: "v"} scheme.AddKnownTypes(testGV, &testPatchType{}) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go index afc95947fe9..2c856c5fec4 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go @@ -35,6 +35,7 @@ import ( "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/finisher" + requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" @@ -69,7 +70,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa return } - body, err := limitedReadBody(req, scope.MaxRequestBodyBytes) + body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Update) trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err}) if err != nil { scope.err(err, w, req) diff --git a/vendor/modules.txt b/vendor/modules.txt index 4abd32f05ef..419a3417967 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1565,6 +1565,7 @@ k8s.io/apiserver/pkg/endpoints/handlers k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal k8s.io/apiserver/pkg/endpoints/handlers/finisher +k8s.io/apiserver/pkg/endpoints/handlers/metrics k8s.io/apiserver/pkg/endpoints/handlers/negotiation k8s.io/apiserver/pkg/endpoints/handlers/responsewriters k8s.io/apiserver/pkg/endpoints/metrics