diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index 1ee6ec34ac8..11c6f905984 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -1133,7 +1133,7 @@ profiles: ConfigFile: unknownFieldConfig, Logs: logs.NewOptions(), }, - expectedError: "found unknown field: foo", + expectedError: `unknown field "foo"`, checkErrFn: runtime.IsStrictDecodingError, }, { diff --git a/pkg/api/testing/serialization_test.go b/pkg/api/testing/serialization_test.go index f5c725998db..31cd68326ad 100644 --- a/pkg/api/testing/serialization_test.go +++ b/pkg/api/testing/serialization_test.go @@ -26,7 +26,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - jsoniter "github.com/json-iterator/go" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -38,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/runtime/serializer/streaming" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" @@ -589,59 +587,6 @@ func BenchmarkDecodeIntoJSON(b *testing.B) { b.StopTimer() } -// BenchmarkDecodeIntoJSONCodecGenConfigFast provides a baseline -// for JSON decode performance with jsoniter.ConfigFast -func BenchmarkDecodeIntoJSONCodecGenConfigFast(b *testing.B) { - kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion) - items := benchmarkItems(b) - width := len(items) - encoded := make([][]byte, width) - for i := range items { - data, err := runtime.Encode(kcodec, &items[i]) - if err != nil { - b.Fatal(err) - } - encoded[i] = data - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - obj := v1.Pod{} - if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil { - b.Fatal(err) - } - } - b.StopTimer() -} - -// BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary provides a -// baseline for JSON decode performance with -// jsoniter.ConfigCompatibleWithStandardLibrary, but with case sensitivity set -// to true -func BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary(b *testing.B) { - kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion) - items := benchmarkItems(b) - width := len(items) - encoded := make([][]byte, width) - for i := range items { - data, err := runtime.Encode(kcodec, &items[i]) - if err != nil { - b.Fatal(err) - } - encoded[i] = data - } - - b.ResetTimer() - iter := json.CaseSensitiveJSONIterator() - for i := 0; i < b.N; i++ { - obj := v1.Pod{} - if err := iter.Unmarshal(encoded[i%width], &obj); err != nil { - b.Fatal(err) - } - } - b.StopTimer() -} - // BenchmarkEncodeYAMLMarshal provides a baseline for regular YAML encode performance func BenchmarkEncodeYAMLMarshal(b *testing.B) { items := benchmarkItems(b) diff --git a/pkg/api/testing/unstructured_test.go b/pkg/api/testing/unstructured_test.go index 21c6373056c..2f0f46d637c 100644 --- a/pkg/api/testing/unstructured_test.go +++ b/pkg/api/testing/unstructured_test.go @@ -19,12 +19,10 @@ package testing import ( "math/rand" "reflect" - "sort" "testing" "github.com/google/go-cmp/cmp" fuzz "github.com/google/gofuzz" - jsoniter "github.com/json-iterator/go" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" @@ -260,75 +258,3 @@ func BenchmarkFromUnstructuredViaJSON(b *testing.B) { } b.StopTimer() } - -func BenchmarkToUnstructuredViaJSONIter(b *testing.B) { - items := benchmarkItems(b) - size := len(items) - var keys []string - for k := range jsonIterConfig { - keys = append(keys, k) - } - sort.Strings(keys) - for _, name := range keys { - c := jsonIterConfig[name] - b.Run(name, func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - data, err := c.Marshal(&items[i%size]) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - unstr := map[string]interface{}{} - if err := c.Unmarshal(data, &unstr); err != nil { - b.Fatalf("unexpected error: %v", err) - } - } - b.StopTimer() - }) - } -} - -var jsonIterConfig = map[string]jsoniter.API{ - "default": jsoniter.ConfigDefault, - "fastest": jsoniter.ConfigFastest, - "compat": jsoniter.ConfigCompatibleWithStandardLibrary, -} - -func BenchmarkFromUnstructuredViaJSONIter(b *testing.B) { - items := benchmarkItems(b) - var unstr []map[string]interface{} - for i := range items { - data, err := jsoniter.Marshal(&items[i]) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - item := map[string]interface{}{} - if err := json.Unmarshal(data, &item); err != nil { - b.Fatalf("unexpected error: %v", err) - } - unstr = append(unstr, item) - } - size := len(items) - var keys []string - for k := range jsonIterConfig { - keys = append(keys, k) - } - sort.Strings(keys) - for _, name := range keys { - c := jsonIterConfig[name] - b.Run(name, func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - item, err := c.Marshal(unstr[i%size]) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - obj := v1.Pod{} - if err := c.Unmarshal(item, &obj); err != nil { - b.Fatalf("unexpected error: %v", err) - } - } - b.StopTimer() - }) - } -} diff --git a/pkg/scheduler/apis/config/scheme/scheme_test.go b/pkg/scheduler/apis/config/scheme/scheme_test.go index fca416ede0c..dbb514c3f7b 100644 --- a/pkg/scheduler/apis/config/scheme/scheme_test.go +++ b/pkg/scheduler/apis/config/scheme/scheme_test.go @@ -263,7 +263,7 @@ profiles: - Score: 2 Utilization: 1 `), - wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.RequestedToCapacityRatio: v1beta2.RequestedToCapacityRatioParam.Shape: []v1beta2.UtilizationShapePoint: v1beta2.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`, + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`, }, { name: "v1beta2 NodeResourcesFitArgs resources encoding is strict", @@ -279,7 +279,7 @@ profiles: - Name: cpu Weight: 1 `), - wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.Resources: []v1beta2.ResourceSpec: v1beta2.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`, + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`, }, { name: "out-of-tree plugin args", @@ -604,7 +604,7 @@ profiles: - Score: 2 Utilization: 1 `), - wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.RequestedToCapacityRatio: v1beta3.RequestedToCapacityRatioParam.Shape: []v1beta3.UtilizationShapePoint: v1beta3.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`, + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`, }, { name: "v1beta3 NodeResourcesFitArgs resources encoding is strict", @@ -620,7 +620,7 @@ profiles: - Name: cpu Weight: 1 `), - wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.Resources: []v1beta3.ResourceSpec: v1beta3.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`, + wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`, }, { name: "out-of-tree plugin args", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce.go index 39850e3d20e..d9c486c37ef 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce.go @@ -22,11 +22,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" ) -var encodingjson = json.CaseSensitiveJSONIterator() - // GetObjectMeta does conversion of JSON to ObjectMeta. It first tries json.Unmarshal into a metav1.ObjectMeta // type. If that does not work and dropMalformedFields is true, it does field-by-field best-effort conversion // throwing away fields which lead to errors. @@ -38,11 +36,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav // round-trip through JSON first, hoping that unmarshalling just works objectMeta := &metav1.ObjectMeta{} - metadataBytes, err := encodingjson.Marshal(metadata) + metadataBytes, err := utiljson.Marshal(metadata) if err != nil { return nil, false, err } - if err = encodingjson.Unmarshal(metadataBytes, objectMeta); err == nil { + if err = utiljson.Unmarshal(metadataBytes, objectMeta); err == nil { // if successful, return return objectMeta, true, nil } @@ -63,11 +61,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav testObjectMeta := &metav1.ObjectMeta{} for k, v := range metadataMap { // serialize a single field - if singleFieldBytes, err := encodingjson.Marshal(map[string]interface{}{k: v}); err == nil { + if singleFieldBytes, err := utiljson.Marshal(map[string]interface{}{k: v}); err == nil { // do a test unmarshal - if encodingjson.Unmarshal(singleFieldBytes, testObjectMeta) == nil { + if utiljson.Unmarshal(singleFieldBytes, testObjectMeta) == nil { // if that succeeds, unmarshal for real - encodingjson.Unmarshal(singleFieldBytes, accumulatedObjectMeta) + utiljson.Unmarshal(singleFieldBytes, accumulatedObjectMeta) } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce_test.go index 8f844752d43..c72240146fe 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/diff" + utiljson "k8s.io/apimachinery/pkg/util/json" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) @@ -114,12 +115,12 @@ func TestMalformedObjectMetaFields(t *testing.T) { } // See if it can unmarshal to object meta - spuriousJSON, err := encodingjson.Marshal(spuriousMetaMap) + spuriousJSON, err := utiljson.Marshal(spuriousMetaMap) if err != nil { t.Fatalf("error on %v=%#v: %v", pth, v, err) } expectedObjectMeta := &metav1.ObjectMeta{} - if err := encodingjson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil { + if err := utiljson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil { // if standard json unmarshal would fail decoding this field, drop the field entirely truncatedMetaMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fuzzedObjectMeta.DeepCopy()) if err != nil { @@ -133,12 +134,12 @@ func TestMalformedObjectMetaFields(t *testing.T) { DeleteJSONPath(truncatedMetaMap, pth[:1], 0) } - truncatedJSON, err := encodingjson.Marshal(truncatedMetaMap) + truncatedJSON, err := utiljson.Marshal(truncatedMetaMap) if err != nil { t.Fatalf("error on %v=%#v: %v", pth, v, err) } expectedObjectMeta = &metav1.ObjectMeta{} - if err := encodingjson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil { + if err := utiljson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil { t.Fatalf("error on %v=%#v: %v", pth, v, err) } } @@ -190,7 +191,7 @@ func TestGetObjectMetaNils(t *testing.T) { } // double check this what the kube JSON decode is doing - bs, _ := encodingjson.Marshal(u.UnstructuredContent()) + bs, _ := utiljson.Marshal(u.UnstructuredContent()) kubeObj, _, err := clientgoscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion).Decode(bs, nil, nil) if err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go index 0ad415cf86e..c50a1902ecc 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" ) type GroupVersionHolder struct { @@ -46,13 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) { if !reflect.DeepEqual(result.GV, c.expect) { t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV) } - // test the json-iterator codec - iter := json.CaseSensitiveJSONIterator() - if err := iter.Unmarshal(c.input, &result); err != nil { - t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err) + // test the utiljson codec + if err := utiljson.Unmarshal(c.input, &result); err != nil { + t.Errorf("util/json codec failed to unmarshal input '%v': %v", c.input, err) } if !reflect.DeepEqual(result.GV, c.expect) { - t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV) + t.Errorf("util/json codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV) } } } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go index 648e8b58a40..f216527ddb7 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" ) func TestVerbsMarshalJSON(t *testing.T) { @@ -56,10 +56,9 @@ func TestVerbsJsonIterUnmarshalJSON(t *testing.T) { {`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}}, } - iter := json.CaseSensitiveJSONIterator() for i, c := range cases { var result APIResource - if err := iter.Unmarshal([]byte(c.input), &result); err != nil { + if err := utiljson.Unmarshal([]byte(c.input), &result); err != nil { t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err) } if !reflect.DeepEqual(result, c.result) { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/converter_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/converter_test.go index 224b2838ba5..a96778628e6 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/converter_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/converter_test.go @@ -274,7 +274,7 @@ func TestRoundTrip(t *testing.T) { { // Test slice of interface{} with different values. obj: &D{ - A: []interface{}{3.0, "3.0", nil}, + A: []interface{}{float64(3.5), int64(4), "3.0", nil}, }, }, } @@ -322,11 +322,11 @@ func TestUnrecognized(t *testing.T) { err error }{ { - data: "{\"da\":[3.0,\"3.0\",null]}", + data: "{\"da\":[3.5,4,\"3.0\",null]}", obj: &D{}, }, { - data: "{\"ea\":[3.0,\"3.0\",null]}", + data: "{\"ea\":[3.5,4,\"3.0\",null]}", obj: &E{}, }, { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/error.go b/staging/src/k8s.io/apimachinery/pkg/runtime/error.go index be0c5edc855..55e8b5ac269 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/error.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/error.go @@ -19,6 +19,7 @@ package runtime import ( "fmt" "reflect" + "strings" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -124,20 +125,26 @@ func IsMissingVersion(err error) bool { // strictDecodingError is a base error type that is returned by a strict Decoder such // as UniversalStrictDecoder. type strictDecodingError struct { - message string - data string + errors []error } // NewStrictDecodingError creates a new strictDecodingError object. -func NewStrictDecodingError(message string, data string) error { +func NewStrictDecodingError(errors []error) error { return &strictDecodingError{ - message: message, - data: data, + errors: errors, } } func (e *strictDecodingError) Error() string { - return fmt.Sprintf("strict decoder error for %s: %s", e.data, e.message) + var s strings.Builder + s.WriteString("strict decoding error: ") + for i, err := range e.errors { + if i != 0 { + s.WriteString(", ") + } + s.WriteString(err.Error()) + } + return s.String() } // IsStrictDecodingError returns true if the error indicates that the provided object diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go index 1bdf5a48f19..daf0ea59665 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go @@ -20,10 +20,8 @@ import ( "encoding/json" "io" "strconv" - "unsafe" - jsoniter "github.com/json-iterator/go" - "github.com/modern-go/reflect2" + kjson "sigs.k8s.io/json" "sigs.k8s.io/yaml" "k8s.io/apimachinery/pkg/runtime" @@ -110,79 +108,6 @@ type Serializer struct { var _ runtime.Serializer = &Serializer{} var _ recognizer.RecognizingDecoder = &Serializer{} -type customNumberExtension struct { - jsoniter.DummyExtension -} - -func (cne *customNumberExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { - if typ.String() == "interface {}" { - return customNumberDecoder{} - } - return nil -} - -type customNumberDecoder struct { -} - -func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - switch iter.WhatIsNext() { - case jsoniter.NumberValue: - var number jsoniter.Number - iter.ReadVal(&number) - i64, err := strconv.ParseInt(string(number), 10, 64) - if err == nil { - *(*interface{})(ptr) = i64 - return - } - f64, err := strconv.ParseFloat(string(number), 64) - if err == nil { - *(*interface{})(ptr) = f64 - return - } - iter.ReportError("DecodeNumber", err.Error()) - default: - *(*interface{})(ptr) = iter.Read() - } -} - -// CaseSensitiveJSONIterator returns a jsoniterator API that's configured to be -// case-sensitive when unmarshalling, and otherwise compatible with -// the encoding/json standard library. -func CaseSensitiveJSONIterator() jsoniter.API { - config := jsoniter.Config{ - EscapeHTML: true, - SortMapKeys: true, - ValidateJsonRawMessage: true, - CaseSensitive: true, - }.Froze() - // Force jsoniter to decode number to interface{} via int64/float64, if possible. - config.RegisterExtension(&customNumberExtension{}) - return config -} - -// StrictCaseSensitiveJSONIterator returns a jsoniterator API that's configured to be -// case-sensitive, but also disallows unknown fields when unmarshalling. It is compatible with -// the encoding/json standard library. -func StrictCaseSensitiveJSONIterator() jsoniter.API { - config := jsoniter.Config{ - EscapeHTML: true, - SortMapKeys: true, - ValidateJsonRawMessage: true, - CaseSensitive: true, - DisallowUnknownFields: true, - }.Froze() - // Force jsoniter to decode number to interface{} via int64/float64, if possible. - config.RegisterExtension(&customNumberExtension{}) - return config -} - -// Private copies of jsoniter to try to shield against possible mutations -// from outside. Still does not protect from package level jsoniter.Register*() functions - someone calling them -// in some other library will mess with every usage of the jsoniter library in the whole program. -// See https://github.com/json-iterator/go/issues/265 -var caseSensitiveJSONIterator = CaseSensitiveJSONIterator() -var strictCaseSensitiveJSONIterator = StrictCaseSensitiveJSONIterator() - // gvkWithDefaults returns group kind and version defaulting from provided default func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind { if len(actual.Kind) == 0 { @@ -237,7 +162,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i types, _, err := s.typer.ObjectKinds(into) switch { case runtime.IsNotRegisteredError(err), isUnstructured: - if err := caseSensitiveJSONIterator.Unmarshal(data, into); err != nil { + if err := kjson.UnmarshalCaseSensitivePreserveInts(data, into); err != nil { return nil, actual, err } return into, actual, nil @@ -261,35 +186,35 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i return nil, actual, err } - if err := caseSensitiveJSONIterator.Unmarshal(data, obj); err != nil { - return nil, actual, err - } - - // If the deserializer is non-strict, return successfully here. + // If the deserializer is non-strict, return here. if !s.options.Strict { + if err := kjson.UnmarshalCaseSensitivePreserveInts(data, obj); err != nil { + return nil, actual, err + } return obj, actual, nil } - // In strict mode pass the data trough the YAMLToJSONStrict converter. - // This is done to catch duplicate fields regardless of encoding (JSON or YAML). For JSON data, - // the output would equal the input, unless there is a parsing error such as duplicate fields. - // As we know this was successful in the non-strict case, the only error that may be returned here - // is because of the newly-added strictness. hence we know we can return the typed strictDecoderError - // the actual error is that the object contains duplicate fields. - altered, err := yaml.YAMLToJSONStrict(originalData) + var allStrictErrs []error + if s.options.Yaml { + // In strict mode pass the original data through the YAMLToJSONStrict converter. + // This is done to catch duplicate fields in YAML that would have been dropped in the original YAMLToJSON conversion. + // TODO: rework YAMLToJSONStrict to return warnings about duplicate fields without terminating so we don't have to do this twice. + _, err := yaml.YAMLToJSONStrict(originalData) + if err != nil { + allStrictErrs = append(allStrictErrs, err) + } + } + + strictJSONErrs, err := kjson.UnmarshalStrict(data, obj) if err != nil { - return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData)) + // fatal decoding error, not due to strictness + return nil, actual, err } - // As performance is not an issue for now for the strict deserializer (one has regardless to do - // the unmarshal twice), we take the sanitized, altered data that is guaranteed to have no duplicated - // fields, and unmarshal this into a copy of the already-populated obj. Any error that occurs here is - // due to that a matching field doesn't exist in the object. hence we can return a typed strictDecoderError, - // the actual error is that the object contains unknown field. - strictObj := obj.DeepCopyObject() - if err := strictCaseSensitiveJSONIterator.Unmarshal(altered, strictObj); err != nil { - return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData)) + allStrictErrs = append(allStrictErrs, strictJSONErrs...) + if len(allStrictErrs) > 0 { + // return the successfully decoded object along with the strict errors + return obj, actual, runtime.NewStrictDecodingError(allStrictErrs) } - // Always return the same object as the non-strict serializer to avoid any deviations. return obj, actual, nil } diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_limit_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_limit_test.go index a741541fb2e..8214d293c62 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_limit_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_limit_test.go @@ -123,7 +123,6 @@ func testcases() []testcase { var decoders = map[string]func([]byte, interface{}) error{ "gojson": gojson.Unmarshal, "utiljson": utiljson.Unmarshal, - "jsoniter": CaseSensitiveJSONIterator().Unmarshal, } func TestJSONLimits(t *testing.T) { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_test.go index aac6cc3f7c2..29a94609edd 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_test.go @@ -39,8 +39,7 @@ type testDecodable struct { Interface interface{} `json:"interface"` } -// DecodableSpec has 15 fields. json-iterator treats struct with more than 10 -// fields differently from struct that has less than 10 fields. +// DecodableSpec has 15 fields. type DecodableSpec struct { A int `json:"A"` B int `json:"B"` @@ -264,7 +263,7 @@ func TestDecode(t *testing.T) { creater: &mockCreater{obj: &testDecodable{}}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, errFn: func(err error) bool { - return strings.Contains(err.Error(), `json_test.testDecodable.Interface: DecodeNumber: strconv.ParseFloat: parsing "1e1000": value out of range`) + return strings.Contains(err.Error(), `json: cannot unmarshal number 1e1000 into Go struct field testDecodable.interface of type float64`) }, }, // Unmarshalling is case-sensitive @@ -298,7 +297,7 @@ func TestDecode(t *testing.T) { typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, errFn: func(err error) bool { - return strings.Contains(err.Error(), "found unknown field") + return strings.Contains(err.Error(), `unknown field "unknown"`) }, strict: true, }, @@ -309,7 +308,7 @@ func TestDecode(t *testing.T) { typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, errFn: func(err error) bool { - return strings.Contains(err.Error(), "found unknown field: unknown") + return strings.Contains(err.Error(), `unknown field "unknown"`) }, yaml: true, strict: true, @@ -321,7 +320,7 @@ func TestDecode(t *testing.T) { typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, errFn: func(err error) bool { - return strings.Contains(err.Error(), `"value" already set in map`) + return strings.Contains(err.Error(), `duplicate field "value"`) }, strict: true, }, @@ -526,7 +525,7 @@ func TestDecode(t *testing.T) { if !test.errFn(err) { t.Errorf("%d: failed: %v", i, err) } - if obj != nil { + if !runtime.IsStrictDecodingError(err) && obj != nil { t.Errorf("%d: should have returned nil object", i) } continue diff --git a/staging/src/k8s.io/apimachinery/pkg/util/json/json.go b/staging/src/k8s.io/apimachinery/pkg/util/json/json.go index 778e58f704b..55dba361c36 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/json/json.go @@ -17,10 +17,11 @@ limitations under the License. package json import ( - "bytes" "encoding/json" "fmt" "io" + + kjson "sigs.k8s.io/json" ) // NewEncoder delegates to json.NewEncoder @@ -38,50 +39,11 @@ func Marshal(v interface{}) ([]byte, error) { // limit recursive depth to prevent stack overflow errors const maxDepth = 10000 -// Unmarshal unmarshals the given data -// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers -// are converted to int64 or float64 +// Unmarshal unmarshals the given data. +// Object keys are case-sensitive. +// Numbers decoded into interface{} fields are converted to int64 or float64. func Unmarshal(data []byte, v interface{}) error { - switch v := v.(type) { - case *map[string]interface{}: - // Build a decoder from the given data - decoder := json.NewDecoder(bytes.NewBuffer(data)) - // Preserve numbers, rather than casting to float64 automatically - decoder.UseNumber() - // Run the decode - if err := decoder.Decode(v); err != nil { - return err - } - // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return ConvertMapNumbers(*v, 0) - - case *[]interface{}: - // Build a decoder from the given data - decoder := json.NewDecoder(bytes.NewBuffer(data)) - // Preserve numbers, rather than casting to float64 automatically - decoder.UseNumber() - // Run the decode - if err := decoder.Decode(v); err != nil { - return err - } - // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return ConvertSliceNumbers(*v, 0) - - case *interface{}: - // Build a decoder from the given data - decoder := json.NewDecoder(bytes.NewBuffer(data)) - // Preserve numbers, rather than casting to float64 automatically - decoder.UseNumber() - // Run the decode - if err := decoder.Decode(v); err != nil { - return err - } - // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return ConvertInterfaceNumbers(v, 0) - - default: - return json.Unmarshal(data, v) - } + return kjson.UnmarshalCaseSensitivePreserveInts(data, v) } // ConvertInterfaceNumbers converts any json.Number values to int64 or float64. diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go index a419328f436..d73b220ec16 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" admissionmetrics "k8s.io/apiserver/pkg/admission/metrics" @@ -61,8 +62,6 @@ const ( MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix ) -var encodingjson = json.CaseSensitiveJSONIterator() - type mutatingDispatcher struct { cm *webhookutil.ClientManager plugin *Plugin @@ -444,7 +443,7 @@ func mutationAnnotationValue(configuration, webhook string, mutated bool) (strin Webhook: webhook, Mutated: mutated, } - bytes, err := encodingjson.Marshal(m) + bytes, err := utiljson.Marshal(m) return string(bytes), err } @@ -455,6 +454,6 @@ func jsonPatchAnnotationValue(configuration, webhook string, patch interface{}) Patch: patch, PatchType: string(admissionv1.PatchTypeJSONPatch), } - bytes, err := encodingjson.Marshal(p) + bytes, err := utiljson.Marshal(p) return string(bytes), err } diff --git a/staging/src/k8s.io/cli-runtime/pkg/resource/builder_test.go b/staging/src/k8s.io/cli-runtime/pkg/resource/builder_test.go index 6a46454195d..bc6e3c66fb0 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/resource/builder_test.go +++ b/staging/src/k8s.io/cli-runtime/pkg/resource/builder_test.go @@ -50,7 +50,7 @@ import ( utiltesting "k8s.io/client-go/util/testing" // TODO we need to remove this linkage and create our own scheme - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" ) @@ -1762,7 +1762,7 @@ func TestUnstructured(t *testing.T) { { name: "badpod", file: "badpod.json", - expectedError: "v1.ObjectMeta.Annotations", + expectedError: "ObjectMeta.", }, } diff --git a/staging/src/k8s.io/cli-runtime/pkg/resource/metadata_decoder.go b/staging/src/k8s.io/cli-runtime/pkg/resource/metadata_decoder.go index 12770573e27..d688c3a08ae 100644 --- a/staging/src/k8s.io/cli-runtime/pkg/resource/metadata_decoder.go +++ b/staging/src/k8s.io/cli-runtime/pkg/resource/metadata_decoder.go @@ -20,12 +20,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" ) -// hold a single instance of the case-sensitive decoder -var caseSensitiveJsonIterator = json.CaseSensitiveJSONIterator() - // metadataValidatingDecoder wraps a decoder and additionally ensures metadata schema fields decode before returning an unstructured object type metadataValidatingDecoder struct { decoder runtime.Decoder @@ -47,7 +44,7 @@ func (m *metadataValidatingDecoder) Decode(data []byte, defaults *schema.GroupVe // make sure the data can decode into ObjectMeta before we return, // so we don't silently truncate schema errors in metadata later with accesser get/set calls v := &metadataOnlyObject{} - if typedErr := caseSensitiveJsonIterator.Unmarshal(data, v); typedErr != nil { + if typedErr := utiljson.Unmarshal(data, v); typedErr != nil { return obj, gvk, typedErr } return obj, gvk, err diff --git a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go index cdafa5a7496..a6c50907e82 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go @@ -22,8 +22,7 @@ import ( "net/http" "strings" - jsoniter "github.com/json-iterator/go" - + utiljson "k8s.io/apimachinery/pkg/util/json" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/kube-openapi/pkg/validation/spec" @@ -80,7 +79,7 @@ func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *sp return nil, "", http.StatusNotFound, nil case http.StatusOK: openAPISpec := &spec.Swagger{} - if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(writer.data, openAPISpec); err != nil { + if err := utiljson.Unmarshal(writer.data, openAPISpec); err != nil { return nil, "", 0, err } newEtag = writer.Header().Get("Etag") diff --git a/staging/src/k8s.io/pod-security-admission/admission/api/load/load_test.go b/staging/src/k8s.io/pod-security-admission/admission/api/load/load_test.go index f01f171dab4..11136c571ed 100644 --- a/staging/src/k8s.io/pod-security-admission/admission/api/load/load_test.go +++ b/staging/src/k8s.io/pod-security-admission/admission/api/load/load_test.go @@ -277,7 +277,7 @@ exemptions: "apiVersion":"pod-security.admission.config.k8s.io/v1alpha1", "kind":"PodSecurityConfiguration", "deflaults":{"enforce":"baseline"}}`), - expectErr: `unknown field: deflaults`, + expectErr: `unknown field "deflaults"`, }, }