diff --git a/pkg/conversion/converter.go b/pkg/conversion/converter.go index c2ab406a69f..d337f831588 100644 --- a/pkg/conversion/converter.go +++ b/pkg/conversion/converter.go @@ -54,6 +54,12 @@ type Converter struct { // Map from a type to a function which applies defaults. defaultingFuncs map[reflect.Type]reflect.Value + // Map from an input type to a function which can apply a key name mapping + inputFieldMappingFuncs map[reflect.Type]FieldMappingFunc + + // Map from an input type to a set of default conversion flags. + inputDefaultFlags map[reflect.Type]FieldMatchingFlags + // If non-nil, will be called to print helpful debugging info. Quite verbose. Debug DebugLogger @@ -71,6 +77,9 @@ func NewConverter() *Converter { nameFunc: func(t reflect.Type) string { return t.Name() }, structFieldDests: map[typeNamePair][]typeNamePair{}, structFieldSources: map[typeNamePair][]typeNamePair{}, + + inputFieldMappingFuncs: map[reflect.Type]FieldMappingFunc{}, + inputDefaultFlags: map[reflect.Type]FieldMatchingFlags{}, } } @@ -98,12 +107,18 @@ type Scope interface { Meta() *Meta } +// FieldMappingFunc can convert an input field value into different values, depending on +// the value of the source or destination struct tags. +type FieldMappingFunc func(key string, sourceTag, destTag reflect.StructTag) (source string, dest string) + // Meta is supplied by Scheme, when it calls Convert. type Meta struct { SrcVersion string DestVersion string - // TODO: If needed, add a user data field here. + // KeyNameMapping is an optional function which may map the listed key (field name) + // into a source and destination value. + KeyNameMapping FieldMappingFunc } // scope contains information about an ongoing conversion. @@ -301,6 +316,21 @@ func (c *Converter) RegisterDefaultingFunc(defaultingFunc interface{}) error { return nil } +// RegisterInputDefaults registers a field name mapping function, used when converting +// from maps to structs. Inputs to the conversion methods are checked for this type and a mapping +// applied automatically if the input matches in. A set of default flags for the input conversion +// may also be provided, which will be used when no explicit flags are requested. +func (c *Converter) RegisterInputDefaults(in interface{}, fn FieldMappingFunc, defaultFlags FieldMatchingFlags) error { + fv := reflect.ValueOf(in) + ft := fv.Type() + if ft.Kind() != reflect.Ptr { + return fmt.Errorf("expected pointer 'in' argument, got: %v", ft) + } + c.inputFieldMappingFuncs[ft] = fn + c.inputDefaultFlags[ft] = defaultFlags + return nil +} + // FieldMatchingFlags contains a list of ways in which struct fields could be // copied. These constants may be | combined. type FieldMatchingFlags int @@ -538,10 +568,16 @@ func (c *Converter) defaultConvert(sv, dv reflect.Value, scope *scope) error { return nil } +var stringType = reflect.TypeOf("") + func toKVValue(v reflect.Value) kvValue { switch v.Kind() { case reflect.Struct: return structAdaptor(v) + case reflect.Map: + if v.Type().Key().AssignableTo(stringType) { + return stringMapAdaptor(v) + } } return nil @@ -561,15 +597,48 @@ type kvValue interface { confirmSet(key string, v reflect.Value) bool } +type stringMapAdaptor reflect.Value + +func (a stringMapAdaptor) len() int { + return reflect.Value(a).Len() +} + +func (a stringMapAdaptor) keys() []string { + v := reflect.Value(a) + keys := make([]string, v.Len()) + for i, v := range v.MapKeys() { + if v.IsNil() { + continue + } + switch t := v.Interface().(type) { + case string: + keys[i] = t + } + } + return keys +} + +func (a stringMapAdaptor) tagOf(key string) reflect.StructTag { + return "" +} + +func (a stringMapAdaptor) value(key string) reflect.Value { + return reflect.Value(a).MapIndex(reflect.ValueOf(key)) +} + +func (a stringMapAdaptor) confirmSet(key string, v reflect.Value) bool { + return true +} + type structAdaptor reflect.Value -func (sa structAdaptor) len() int { - v := reflect.Value(sa) +func (a structAdaptor) len() int { + v := reflect.Value(a) return v.Type().NumField() } -func (sa structAdaptor) keys() []string { - v := reflect.Value(sa) +func (a structAdaptor) keys() []string { + v := reflect.Value(a) t := v.Type() keys := make([]string, t.NumField()) for i := range keys { @@ -578,8 +647,8 @@ func (sa structAdaptor) keys() []string { return keys } -func (sa structAdaptor) tagOf(key string) reflect.StructTag { - v := reflect.Value(sa) +func (a structAdaptor) tagOf(key string) reflect.StructTag { + v := reflect.Value(a) field, ok := v.Type().FieldByName(key) if ok { return field.Tag @@ -587,12 +656,12 @@ func (sa structAdaptor) tagOf(key string) reflect.StructTag { return "" } -func (sa structAdaptor) value(key string) reflect.Value { - v := reflect.Value(sa) +func (a structAdaptor) value(key string) reflect.Value { + v := reflect.Value(a) return v.FieldByName(key) } -func (sa structAdaptor) confirmSet(key string, v reflect.Value) bool { +func (a structAdaptor) confirmSet(key string, v reflect.Value) bool { return true } @@ -608,6 +677,12 @@ func (c *Converter) convertKV(skv, dkv kvValue, scope *scope) error { if scope.flags.IsSet(SourceToDest) { lister = skv } + + var mapping FieldMappingFunc + if scope.meta != nil && scope.meta.KeyNameMapping != nil { + mapping = scope.meta.KeyNameMapping + } + for _, key := range lister.keys() { if found, err := c.checkField(key, skv, dkv, scope); found { if err != nil { @@ -615,23 +690,31 @@ func (c *Converter) convertKV(skv, dkv kvValue, scope *scope) error { } continue } - df := dkv.value(key) - sf := skv.value(key) + stag := skv.tagOf(key) + dtag := dkv.tagOf(key) + skey := key + dkey := key + if mapping != nil { + skey, dkey = scope.meta.KeyNameMapping(key, stag, dtag) + } + + df := dkv.value(dkey) + sf := skv.value(skey) if !df.IsValid() || !sf.IsValid() { switch { case scope.flags.IsSet(IgnoreMissingFields): // No error. case scope.flags.IsSet(SourceToDest): - return scope.error("%v not present in dest", key) + return scope.error("%v not present in dest", dkey) default: - return scope.error("%v not present in src", key) + return scope.error("%v not present in src", skey) } continue } - scope.srcStack.top().key = key - scope.srcStack.top().tag = skv.tagOf(key) - scope.destStack.top().key = key - scope.destStack.top().tag = dkv.tagOf(key) + scope.srcStack.top().key = skey + scope.srcStack.top().tag = stag + scope.destStack.top().key = dkey + scope.destStack.top().tag = dtag if err := c.convert(sf, df, scope); err != nil { return err } diff --git a/pkg/conversion/converter_test.go b/pkg/conversion/converter_test.go index 4f807760eac..d400da1ecb3 100644 --- a/pkg/conversion/converter_test.go +++ b/pkg/conversion/converter_test.go @@ -20,6 +20,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "testing" "github.com/google/gofuzz" @@ -173,6 +174,105 @@ func TestConverter_CallsRegisteredFunctions(t *testing.T) { } } +func TestConverter_MapsStringArrays(t *testing.T) { + type A struct { + Foo string + Baz int + Other string + } + c := NewConverter() + c.Debug = t + if err := c.RegisterConversionFunc(func(input *[]string, out *string, s Scope) error { + if len(*input) == 0 { + *out = "" + } + *out = (*input)[0] + return nil + }); err != nil { + t.Fatalf("unexpected error %v", err) + } + + x := map[string][]string{ + "Foo": {"bar"}, + "Baz": {"1"}, + "Other": {"", "test"}, + "other": {"wrong"}, + } + y := A{"test", 2, "something"} + + if err := c.Convert(&x, &y, AllowDifferentFieldTypeNames, nil); err == nil { + t.Error("unexpected non-error") + } + + if err := c.RegisterConversionFunc(func(input *[]string, out *int, s Scope) error { + if len(*input) == 0 { + *out = 0 + } + str := (*input)[0] + i, err := strconv.Atoi(str) + if err != nil { + return err + } + *out = i + return nil + }); err != nil { + t.Fatalf("unexpected error %v", err) + } + + if err := c.Convert(&x, &y, AllowDifferentFieldTypeNames, nil); err != nil { + t.Fatalf("unexpected error %v", err) + } + if !reflect.DeepEqual(y, A{"bar", 1, ""}) { + t.Errorf("unexpected result: %#v", y) + } +} + +func TestConverter_MapsStringArraysWithMappingKey(t *testing.T) { + type A struct { + Foo string `json:"test"` + Baz int + Other string + } + c := NewConverter() + c.Debug = t + if err := c.RegisterConversionFunc(func(input *[]string, out *string, s Scope) error { + if len(*input) == 0 { + *out = "" + } + *out = (*input)[0] + return nil + }); err != nil { + t.Fatalf("unexpected error %v", err) + } + + x := map[string][]string{ + "Foo": {"bar"}, + "test": {"baz"}, + } + y := A{"", 0, ""} + + if err := c.Convert(&x, &y, AllowDifferentFieldTypeNames|IgnoreMissingFields, &Meta{}); err != nil { + t.Fatalf("unexpected error %v", err) + } + if !reflect.DeepEqual(y, A{"bar", 0, ""}) { + t.Errorf("unexpected result: %#v", y) + } + + mapping := func(key string, sourceTag, destTag reflect.StructTag) (source string, dest string) { + if s := destTag.Get("json"); len(s) > 0 { + return strings.SplitN(s, ",", 2)[0], key + } + return key, key + } + + if err := c.Convert(&x, &y, AllowDifferentFieldTypeNames|IgnoreMissingFields, &Meta{KeyNameMapping: mapping}); err != nil { + t.Fatalf("unexpected error %v", err) + } + if !reflect.DeepEqual(y, A{"baz", 0, ""}) { + t.Errorf("unexpected result: %#v", y) + } +} + func TestConverter_fuzz(t *testing.T) { // Use the same types from the scheme test. table := []struct { diff --git a/pkg/conversion/decode.go b/pkg/conversion/decode.go index 1c98fae9bbf..34418a59177 100644 --- a/pkg/conversion/decode.go +++ b/pkg/conversion/decode.go @@ -58,7 +58,8 @@ func (s *Scheme) Decode(data []byte) (interface{}, error) { if err != nil { return nil, err } - if err := s.converter.Convert(obj, objOut, 0, s.generateConvertMeta(version, s.InternalVersion)); err != nil { + flags, meta := s.generateConvertMeta(version, s.InternalVersion, obj) + if err := s.converter.Convert(obj, objOut, flags, meta); err != nil { return nil, err } obj = objOut @@ -101,7 +102,8 @@ func (s *Scheme) DecodeInto(data []byte, obj interface{}) error { if err := json.Unmarshal(data, external); err != nil { return err } - if err := s.converter.Convert(external, obj, 0, s.generateConvertMeta(dataVersion, objVersion)); err != nil { + flags, meta := s.generateConvertMeta(dataVersion, objVersion, external) + if err := s.converter.Convert(external, obj, flags, meta); err != nil { return err } diff --git a/pkg/conversion/encode.go b/pkg/conversion/encode.go index e954dad16c2..fa753e0d3a2 100644 --- a/pkg/conversion/encode.go +++ b/pkg/conversion/encode.go @@ -67,7 +67,8 @@ func (s *Scheme) EncodeToVersion(obj interface{}, destVersion string) (data []by if err != nil { return nil, err } - err = s.converter.Convert(obj, objOut, 0, s.generateConvertMeta(objVersion, destVersion)) + flags, meta := s.generateConvertMeta(objVersion, destVersion, obj) + err = s.converter.Convert(obj, objOut, flags, meta) if err != nil { return nil, err } diff --git a/pkg/conversion/scheme.go b/pkg/conversion/scheme.go index 15adea40446..5314d0995bb 100644 --- a/pkg/conversion/scheme.go +++ b/pkg/conversion/scheme.go @@ -232,6 +232,14 @@ func (s *Scheme) AddDefaultingFuncs(defaultingFuncs ...interface{}) error { return nil } +// RegisterInputDefaults sets the provided field mapping function and field matching +// as the defaults for the provided input type. The fn may be nil, in which case no +// mapping will happen by default. Use this method to register a mechanism for handling +// a specific input type in conversion, such as a map[string]string to structs. +func (s *Scheme) RegisterInputDefaults(in interface{}, fn FieldMappingFunc, defaultFlags FieldMatchingFlags) error { + return s.converter.RegisterInputDefaults(in, fn, defaultFlags) +} + // Convert will attempt to convert in into out. Both must be pointers. For easy // testing of conversion functions. Returns an error if the conversion isn't // possible. You can call this with types that haven't been registered (for example, @@ -247,7 +255,11 @@ func (s *Scheme) Convert(in, out interface{}) error { if v, _, err := s.ObjectVersionAndKind(out); err == nil { outVersion = v } - return s.converter.Convert(in, out, AllowDifferentFieldTypeNames, s.generateConvertMeta(inVersion, outVersion)) + flags, meta := s.generateConvertMeta(inVersion, outVersion, in) + if flags == 0 { + flags = AllowDifferentFieldTypeNames + } + return s.converter.Convert(in, out, flags, meta) } // ConvertToVersion attempts to convert an input object to its matching Kind in another @@ -279,7 +291,8 @@ func (s *Scheme) ConvertToVersion(in interface{}, outVersion string) (interface{ return nil, err } - if err := s.converter.Convert(in, out, 0, s.generateConvertMeta(inVersion, outVersion)); err != nil { + flags, meta := s.generateConvertMeta(inVersion, outVersion, in) + if err := s.converter.Convert(in, out, flags, meta); err != nil { return nil, err } @@ -290,11 +303,18 @@ func (s *Scheme) ConvertToVersion(in interface{}, outVersion string) (interface{ return out, nil } +// Converter allows access to the converter for the scheme +func (s *Scheme) Converter() *Converter { + return s.converter +} + // generateConvertMeta constructs the meta value we pass to Convert. -func (s *Scheme) generateConvertMeta(srcVersion, destVersion string) *Meta { - return &Meta{ - SrcVersion: srcVersion, - DestVersion: destVersion, +func (s *Scheme) generateConvertMeta(srcVersion, destVersion string, in interface{}) (FieldMatchingFlags, *Meta) { + t := reflect.TypeOf(in) + return s.converter.inputDefaultFlags[t], &Meta{ + SrcVersion: srcVersion, + DestVersion: destVersion, + KeyNameMapping: s.converter.inputFieldMappingFuncs[t], } } diff --git a/pkg/runtime/conversion.go b/pkg/runtime/conversion.go new file mode 100644 index 00000000000..b47f9e6bbe7 --- /dev/null +++ b/pkg/runtime/conversion.go @@ -0,0 +1,78 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "reflect" + "strconv" + "strings" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion" +) + +// JSONKeyMapper uses the struct tags on a conversion to determine the key value for +// the other side. Use when mapping from a map[string]* to a struct or vice versa. +func JSONKeyMapper(key string, sourceTag, destTag reflect.StructTag) (string, string) { + if s := destTag.Get("json"); len(s) > 0 { + return strings.SplitN(s, ",", 2)[0], key + } + if s := sourceTag.Get("json"); len(s) > 0 { + return key, strings.SplitN(s, ",", 2)[0] + } + return key, key +} + +// DefaultStringConversions are helpers for converting []string and string to real values. +var DefaultStringConversions = []interface{}{ + convertStringSliceToString, + convertStringSliceToInt, + convertStringSliceToInt64, +} + +func convertStringSliceToString(input *[]string, out *string, s conversion.Scope) error { + if len(*input) == 0 { + *out = "" + } + *out = (*input)[0] + return nil +} + +func convertStringSliceToInt(input *[]string, out *int, s conversion.Scope) error { + if len(*input) == 0 { + *out = 0 + } + str := (*input)[0] + i, err := strconv.Atoi(str) + if err != nil { + return err + } + *out = i + return nil +} + +func convertStringSliceToInt64(input *[]string, out *int64, s conversion.Scope) error { + if len(*input) == 0 { + *out = 0 + } + str := (*input)[0] + i, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return err + } + *out = i + return nil +} diff --git a/pkg/runtime/conversion_test.go b/pkg/runtime/conversion_test.go new file mode 100644 index 00000000000..a5a2dcfe179 --- /dev/null +++ b/pkg/runtime/conversion_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime_test + +import ( + "reflect" + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" +) + +type InternalComplex struct { + TypeMeta + String string + Integer int + Integer64 int64 + Int64 int64 +} + +type ExternalComplex struct { + TypeMeta `json:",inline"` + String string `json:"string" description:"testing"` + Integer int `json:"int"` + Integer64 int64 `json:",omitempty"` + Int64 int64 +} + +func (*InternalComplex) IsAnAPIObject() {} +func (*ExternalComplex) IsAnAPIObject() {} + +func TestStringMapConversion(t *testing.T) { + scheme := runtime.NewScheme() + scheme.Log(t) + scheme.AddKnownTypeWithName("", "Complex", &InternalComplex{}) + scheme.AddKnownTypeWithName("external", "Complex", &ExternalComplex{}) + + testCases := map[string]struct { + input map[string][]string + errFn func(error) bool + expected runtime.Object + }{ + "ignores omitempty": { + input: map[string][]string{ + "String": {"not_used"}, + "string": {"value"}, + "int": {"1"}, + "Integer64": {"2"}, + }, + expected: &ExternalComplex{String: "value", Integer: 1}, + }, + "returns error on bad int": { + input: map[string][]string{ + "int": {"a"}, + }, + errFn: func(err error) bool { return err != nil }, + expected: &ExternalComplex{}, + }, + "parses int64": { + input: map[string][]string{ + "Int64": {"-1"}, + }, + expected: &ExternalComplex{Int64: -1}, + }, + "returns error on bad int64": { + input: map[string][]string{ + "Int64": {"a"}, + }, + errFn: func(err error) bool { return err != nil }, + expected: &ExternalComplex{}, + }, + } + + for k, tc := range testCases { + out := &ExternalComplex{} + if err := scheme.Convert(&tc.input, out); (tc.errFn == nil && err != nil) || (tc.errFn != nil && !tc.errFn(err)) { + t.Errorf("%s: unexpected error: %v", k, err) + continue + } else if err != nil { + continue + } + if !reflect.DeepEqual(out, tc.expected) { + t.Errorf("%s: unexpected output: %#v", k, out) + } + } +} diff --git a/pkg/runtime/helper.go b/pkg/runtime/helper.go index 3f3427fd08b..3d4df08091e 100644 --- a/pkg/runtime/helper.go +++ b/pkg/runtime/helper.go @@ -23,6 +23,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion" ) +// TODO: move me to pkg/api/meta func IsListType(obj Object) bool { _, err := GetItemsPtr(obj) return err == nil @@ -32,6 +33,7 @@ func IsListType(obj Object) bool { // If 'list' doesn't have an Items member, it's not really a list type // and an error will be returned. // This function will either return a pointer to a slice, or an error, but not both. +// TODO: move me to pkg/api/meta func GetItemsPtr(list Object) (interface{}, error) { v, err := conversion.EnforcePtr(list) if err != nil { @@ -57,6 +59,7 @@ func GetItemsPtr(list Object) (interface{}, error) { // ExtractList returns obj's Items element as an array of runtime.Objects. // Returns an error if obj is not a List type (does not have an Items member). +// TODO: move me to pkg/api/meta func ExtractList(obj Object) ([]Object, error) { itemsPtr, err := GetItemsPtr(obj) if err != nil { @@ -90,6 +93,7 @@ var objectSliceType = reflect.TypeOf([]Object{}) // objects. // Returns an error if list is not a List type (does not have an Items member), // or if any of the objects are not of the right type. +// TODO: move me to pkg/api/meta func SetList(list Object, objects []Object) error { itemsPtr, err := GetItemsPtr(list) if err != nil { diff --git a/pkg/runtime/scheme.go b/pkg/runtime/scheme.go index 253c4957f88..7fc233767e7 100644 --- a/pkg/runtime/scheme.go +++ b/pkg/runtime/scheme.go @@ -217,6 +217,13 @@ func NewScheme() *Scheme { ); err != nil { panic(err) } + // Enable map[string][]string conversions by default + if err := s.raw.AddConversionFuncs(DefaultStringConversions...); err != nil { + panic(err) + } + if err := s.raw.RegisterInputDefaults(&map[string][]string{}, JSONKeyMapper, conversion.AllowDifferentFieldTypeNames|conversion.IgnoreMissingFields); err != nil { + panic(err) + } return s }