Add maxLength/maxItems/maxProperties support to cel.Compile.
This commit is contained in:
Kermit Alexander
@@ -22,6 +22,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/cel-go/cel"
|
||||
"github.com/google/cel-go/checker"
|
||||
"github.com/google/cel-go/checker/decls"
|
||||
expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -54,10 +55,11 @@ const (
|
||||
type CompilationResult struct {
|
||||
Program cel.Program
|
||||
Error *Error
|
||||
|
||||
// If true, the compiled expression contains a reference to the identifier "oldSelf", and its corresponding rule
|
||||
// is implicitly a transition rule.
|
||||
TransitionRule bool
|
||||
// Represents the worst-case cost of the compiled expression in terms of CEL's cost units, as used by cel.EstimateCost.
|
||||
MaxCost uint64
|
||||
}
|
||||
|
||||
// Compile compiles all the XValidations rules (without recursing into the schema) and returns a slice containing a
|
||||
@@ -111,17 +113,17 @@ func Compile(s *schema.Structural, isResourceRoot bool, perCallLimit uint64) ([]
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
estimator := celEstimator{root: root}
|
||||
// compResults is the return value which saves a list of compilation results in the same order as x-kubernetes-validations rules.
|
||||
compResults := make([]CompilationResult, len(celRules))
|
||||
for i, rule := range celRules {
|
||||
compResults[i] = compileRule(rule, env, perCallLimit)
|
||||
compResults[i] = compileRule(rule, env, perCallLimit, &estimator)
|
||||
}
|
||||
|
||||
return compResults, nil
|
||||
}
|
||||
|
||||
func compileRule(rule apiextensions.ValidationRule, env *cel.Env, perCallLimit uint64) (compilationResult CompilationResult) {
|
||||
func compileRule(rule apiextensions.ValidationRule, env *cel.Env, perCallLimit uint64, estimator *celEstimator) (compilationResult CompilationResult) {
|
||||
if len(strings.TrimSpace(rule.Rule)) == 0 {
|
||||
// include a compilation result, but leave both program and error nil per documented return semantics of this
|
||||
// function
|
||||
@@ -156,7 +158,12 @@ func compileRule(rule apiextensions.ValidationRule, env *cel.Env, perCallLimit u
|
||||
compilationResult.Error = &Error{ErrorTypeInvalid, "program instantiation failed: " + err.Error()}
|
||||
return
|
||||
}
|
||||
|
||||
costEst, err := env.EstimateCost(ast, estimator)
|
||||
if err != nil {
|
||||
compilationResult.Error = &Error{ErrorTypeInternal, "cost estimation failed: " + err.Error()}
|
||||
return
|
||||
}
|
||||
compilationResult.MaxCost = costEst.Max
|
||||
compilationResult.Program = prog
|
||||
return
|
||||
}
|
||||
@@ -168,3 +175,45 @@ func compileRule(rule apiextensions.ValidationRule, env *cel.Env, perCallLimit u
|
||||
func generateUniqueSelfTypeName() string {
|
||||
return fmt.Sprintf("selfType%d", time.Now().Nanosecond())
|
||||
}
|
||||
|
||||
type celEstimator struct {
|
||||
root *celmodel.DeclType
|
||||
}
|
||||
|
||||
func (c *celEstimator) EstimateSize(element checker.AstNode) *checker.SizeEstimate {
|
||||
if len(element.Path()) == 0 {
|
||||
// Path() can return an empty list, early exit if it does since we can't
|
||||
// provide size estimates when that happens
|
||||
return nil
|
||||
}
|
||||
currentNode := c.root
|
||||
// cut off "self" from path, since we always start there
|
||||
for _, name := range element.Path()[1:] {
|
||||
switch name {
|
||||
case "@items", "@values":
|
||||
if currentNode.ElemType == nil {
|
||||
return nil
|
||||
}
|
||||
currentNode = currentNode.ElemType
|
||||
case "@keys":
|
||||
if currentNode.KeyType == nil {
|
||||
return nil
|
||||
}
|
||||
currentNode = currentNode.KeyType
|
||||
default:
|
||||
field, ok := currentNode.Fields[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if field.Type == nil {
|
||||
return nil
|
||||
}
|
||||
currentNode = field.Type
|
||||
}
|
||||
}
|
||||
return &checker.SizeEstimate{Min: 0, Max: uint64(currentNode.MaxElements)}
|
||||
}
|
||||
|
||||
func (c *celEstimator) EstimateCallCost(function, overloadID string, target *checker.AstNode, args []checker.AstNode) *checker.CallEstimate {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ package cel
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -25,6 +26,10 @@ import (
|
||||
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
costLimit = 100000000
|
||||
)
|
||||
|
||||
type validationMatcher interface {
|
||||
matches(cr CompilationResult) bool
|
||||
String() string
|
||||
@@ -658,3 +663,864 @@ func TestCelCompilation(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// take a single rule type in (string/number/map/etc.) and return appropriate values for
|
||||
// Type, Format, and XIntOrString
|
||||
func parseRuleType(ruleType string) (string, string, bool) {
|
||||
if ruleType == "duration" || ruleType == "date" || ruleType == "date-time" {
|
||||
return "string", ruleType, false
|
||||
}
|
||||
if ruleType == "int-or-string" {
|
||||
return "", "", true
|
||||
}
|
||||
return ruleType, "", false
|
||||
}
|
||||
|
||||
func genArrayWithRule(arrayType, rule string) func(maxItems *int64) *schema.Structural {
|
||||
passedType, passedFormat, xIntString := parseRuleType(arrayType)
|
||||
return func(maxItems *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: passedType,
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: passedFormat,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XIntOrString: xIntString,
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxItems,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genArrayOfArraysWithRule(arrayType, rule string) func(maxItems *int64) *schema.Structural {
|
||||
return func(maxItems *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: arrayType,
|
||||
},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxItems,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genObjectArrayWithRule(rule string) func(maxItems *int64) *schema.Structural {
|
||||
return func(maxItems *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"required": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"optional": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"required"},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxItems,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getMapArrayWithRule(mapType, rule string) func(maxItems *int64) *schema.Structural {
|
||||
return func(maxItems *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: mapType,
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxItems,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genMapWithRule(mapType, rule string) func(maxProperties *int64) *schema.Structural {
|
||||
passedType, passedFormat, xIntString := parseRuleType(mapType)
|
||||
return func(maxProperties *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: passedType,
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: passedFormat,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XIntOrString: xIntString,
|
||||
},
|
||||
}},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxProperties: maxProperties,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genStringWithRule(rule string) func(maxLength *int64) *schema.Structural {
|
||||
return func(maxLength *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxLength: maxLength,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genBytesWithRule(rule string) func(maxLength *int64) *schema.Structural {
|
||||
return func(maxLength *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxLength: maxLength,
|
||||
Format: "byte",
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genNestedSpecWithRule(rule string) func(maxLength *int64) *schema.Structural {
|
||||
return func(maxLength *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxLength: maxLength,
|
||||
},
|
||||
}},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxProperties: maxLength,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genAllMaxNestedSpecWithRootRule(rule string) func(maxLength *int64) *schema.Structural {
|
||||
return func(maxLength *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"required"},
|
||||
MaxProperties: maxLength,
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"required": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"optional": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxProperties: maxLength,
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxLength,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genOneMaxNestedSpecWithRootRule(rule string) func(maxLength *int64) *schema.Structural {
|
||||
return func(maxLength *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"required"},
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"required": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"optional": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxProperties: maxLength,
|
||||
},
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func genObjectForMap() *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"required": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"optional": {
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"required"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func genArrayForMap() *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "number",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func genMapForMap() *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "number",
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func genMapWithCustomItemRule(item *schema.Structural, rule string) func(maxProperties *int64) *schema.Structural {
|
||||
return func(maxProperties *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: item},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxProperties: maxProperties,
|
||||
},
|
||||
Extensions: schema.Extensions{
|
||||
XValidations: apiextensions.ValidationRules{
|
||||
{
|
||||
Rule: rule,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func schemaChecker(schema *schema.Structural, expectedCost uint64, calcLimit uint64, t *testing.T) func(t *testing.T) {
|
||||
return func(t *testing.T) {
|
||||
// TODO(DangerOnTheRanger): if perCallLimit in compilation.go changes, this needs to change as well
|
||||
compilationResults, err := Compile(schema, false, uint64(math.MaxInt64))
|
||||
if err != nil {
|
||||
t.Errorf("Expected no error, got: %v", err)
|
||||
}
|
||||
if len(compilationResults) != 1 {
|
||||
t.Errorf("Expected one rule, got: %d", len(compilationResults))
|
||||
}
|
||||
result := compilationResults[0]
|
||||
if result.Error != nil {
|
||||
t.Errorf("Expected no compile-time error, got: %v", result.Error)
|
||||
}
|
||||
if calcLimit == 0 {
|
||||
if result.MaxCost != expectedCost {
|
||||
t.Errorf("Wrong cost (expected %d, got %d)", expectedCost, result.MaxCost)
|
||||
}
|
||||
} else {
|
||||
if result.MaxCost < calcLimit {
|
||||
t.Errorf("Cost did not exceed limit as expected (expected more than %d, got %d)", calcLimit, result.MaxCost)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostEstimation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
schemaGenerator func(maxLength *int64) *schema.Structural
|
||||
expectedCalcCost uint64
|
||||
setMaxElements int64
|
||||
expectedSetCost uint64
|
||||
expectCalcCostExceedsLimit uint64
|
||||
}{
|
||||
{
|
||||
name: "number array with all",
|
||||
schemaGenerator: genArrayWithRule("number", "self.all(x, true)"),
|
||||
expectedCalcCost: 4718591,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 32,
|
||||
},
|
||||
{
|
||||
name: "string array with all",
|
||||
schemaGenerator: genArrayWithRule("string", "self.all(x, true)"),
|
||||
expectedCalcCost: 3145727,
|
||||
setMaxElements: 20,
|
||||
expectedSetCost: 62,
|
||||
},
|
||||
{
|
||||
name: "boolean array with all",
|
||||
schemaGenerator: genArrayWithRule("boolean", "self.all(x, true)"),
|
||||
expectedCalcCost: 1887437,
|
||||
setMaxElements: 5,
|
||||
expectedSetCost: 17,
|
||||
},
|
||||
// all array-of-array tests should have the same expected cost along the same expression,
|
||||
// since arrays-of-arrays are serialized the same in minimized form regardless of item type
|
||||
// of the subarray ([[], [], ...])
|
||||
{
|
||||
name: "array of number arrays with all",
|
||||
schemaGenerator: genArrayOfArraysWithRule("number", "self.all(x, true)"),
|
||||
expectedCalcCost: 3145727,
|
||||
setMaxElements: 100,
|
||||
expectedSetCost: 302,
|
||||
},
|
||||
{
|
||||
name: "array of objects with all",
|
||||
schemaGenerator: genObjectArrayWithRule("self.all(x, true)"),
|
||||
expectedCalcCost: 555128,
|
||||
setMaxElements: 50,
|
||||
expectedSetCost: 152,
|
||||
},
|
||||
{
|
||||
name: "map of numbers with all",
|
||||
schemaGenerator: genMapWithRule("number", "self.all(x, true)"),
|
||||
expectedCalcCost: 1348169,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 32,
|
||||
},
|
||||
{
|
||||
name: "map of numbers with has",
|
||||
schemaGenerator: genMapWithRule("number", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 100,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of strings with all",
|
||||
schemaGenerator: genMapWithRule("string", "self.all(x, true)"),
|
||||
expectedCalcCost: 1179647,
|
||||
setMaxElements: 3,
|
||||
expectedSetCost: 11,
|
||||
},
|
||||
{
|
||||
name: "map of strings with has",
|
||||
schemaGenerator: genMapWithRule("string", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 550,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of booleans with all",
|
||||
schemaGenerator: genMapWithRule("boolean", "self.all(x, true)"),
|
||||
expectedCalcCost: 943718,
|
||||
setMaxElements: 100,
|
||||
expectedSetCost: 302,
|
||||
},
|
||||
{
|
||||
name: "map of booleans with has",
|
||||
schemaGenerator: genMapWithRule("boolean", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 1024,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "string with contains",
|
||||
schemaGenerator: genStringWithRule("self.contains('test')"),
|
||||
expectedCalcCost: 314574,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 5,
|
||||
},
|
||||
{
|
||||
name: "string with startsWith",
|
||||
schemaGenerator: genStringWithRule("self.startsWith('test')"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 15,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "string with endsWith",
|
||||
schemaGenerator: genStringWithRule("self.endsWith('test')"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 30,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "concat string",
|
||||
schemaGenerator: genStringWithRule(`size(self + "hello") > size("hello")`),
|
||||
expectedCalcCost: 314578,
|
||||
setMaxElements: 4,
|
||||
expectedSetCost: 7,
|
||||
},
|
||||
{
|
||||
name: "index of array with numbers",
|
||||
schemaGenerator: genArrayWithRule("number", "self[1] == 0.0"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 5000,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "index of array with strings",
|
||||
schemaGenerator: genArrayWithRule("string", "self[1] == self[1]"),
|
||||
expectedCalcCost: 314577,
|
||||
setMaxElements: 8,
|
||||
expectedSetCost: 314577,
|
||||
},
|
||||
{
|
||||
name: "O(n^2) loop with numbers",
|
||||
schemaGenerator: genArrayWithRule("number", "self.all(x, self.all(y, true))"),
|
||||
expectedCalcCost: 9895601504256,
|
||||
expectCalcCostExceedsLimit: costLimit,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 352,
|
||||
},
|
||||
{
|
||||
name: "O(n^3) loop with numbers",
|
||||
schemaGenerator: genArrayWithRule("number", "self.all(x, self.all(y, self.all(z, true)))"),
|
||||
expectedCalcCost: 13499986500008999998,
|
||||
expectCalcCostExceedsLimit: costLimit,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 3552,
|
||||
},
|
||||
{
|
||||
name: "regex matches simple",
|
||||
schemaGenerator: genStringWithRule(`self.matches("x")`),
|
||||
expectedCalcCost: 314574,
|
||||
setMaxElements: 50,
|
||||
expectedSetCost: 22,
|
||||
},
|
||||
{
|
||||
name: "regex matches empty string",
|
||||
schemaGenerator: genStringWithRule(`"".matches("(((((((((())))))))))[0-9]")`),
|
||||
expectedCalcCost: 7,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 7,
|
||||
},
|
||||
{
|
||||
name: "regex matches empty regex",
|
||||
schemaGenerator: genStringWithRule(`self.matches("")`),
|
||||
expectedCalcCost: 1,
|
||||
setMaxElements: 100,
|
||||
expectedSetCost: 1,
|
||||
},
|
||||
{
|
||||
name: "map of strings with value length",
|
||||
schemaGenerator: genNestedSpecWithRule("self.all(x, x.contains(self[x]))"),
|
||||
expectedCalcCost: 2752507,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 72,
|
||||
},
|
||||
{
|
||||
name: "set array maxLength to zero",
|
||||
schemaGenerator: genArrayWithRule("number", "self[3] == 0.0"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 0,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "set map maxLength to zero",
|
||||
schemaGenerator: genMapWithRule("number", `self["x"] == 0.0`),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 0,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "set string maxLength to zero",
|
||||
schemaGenerator: genStringWithRule(`self == "x"`),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 0,
|
||||
expectedSetCost: 1,
|
||||
},
|
||||
{
|
||||
name: "set bytes maxLength to zero",
|
||||
schemaGenerator: genBytesWithRule(`self == b"x"`),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 0,
|
||||
expectedSetCost: 1,
|
||||
},
|
||||
{
|
||||
name: "set maxLength greater than estimated maxLength",
|
||||
schemaGenerator: genArrayWithRule("number", "self.all(x, x == 0.0)"),
|
||||
expectedCalcCost: 6291454,
|
||||
setMaxElements: 3 * 1024 * 2048,
|
||||
expectedSetCost: 25165826,
|
||||
},
|
||||
{
|
||||
name: "nested types with root rule with all supporting maxLength",
|
||||
schemaGenerator: genAllMaxNestedSpecWithRootRule(`self.all(x, x["y"].required == "z")`),
|
||||
expectedCalcCost: 7340027,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 72,
|
||||
},
|
||||
{
|
||||
name: "nested types with root rule with one supporting maxLength",
|
||||
schemaGenerator: genOneMaxNestedSpecWithRootRule(`self.all(x, x["y"].required == "z")`),
|
||||
expectedCalcCost: 7340027,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 7340027,
|
||||
},
|
||||
{
|
||||
name: "int-or-string array with all",
|
||||
schemaGenerator: genArrayWithRule("int-or-string", "self.all(x, true)"),
|
||||
expectedCalcCost: 4718591,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 32,
|
||||
},
|
||||
{
|
||||
name: "index of array with int-or-strings",
|
||||
schemaGenerator: genArrayWithRule("int-or-string", "self[0] == 5"),
|
||||
expectedCalcCost: 3,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 3,
|
||||
},
|
||||
{
|
||||
name: "index of array with booleans",
|
||||
schemaGenerator: genArrayWithRule("boolean", "self[0] == false"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 25,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "index of array of objects",
|
||||
schemaGenerator: genObjectArrayWithRule("self[0] == null"),
|
||||
expectedCalcCost: 2,
|
||||
setMaxElements: 422,
|
||||
expectedSetCost: 2,
|
||||
},
|
||||
{
|
||||
name: "index of array of array of numnbers",
|
||||
schemaGenerator: genArrayOfArraysWithRule("number", "self[0][0] == -1.0"),
|
||||
expectedCalcCost: 3,
|
||||
setMaxElements: 51,
|
||||
expectedSetCost: 3,
|
||||
},
|
||||
{
|
||||
name: "array of number maps with all",
|
||||
schemaGenerator: getMapArrayWithRule("number", `self.all(x, x.y == 25.2)`),
|
||||
expectedCalcCost: 6291452,
|
||||
setMaxElements: 12,
|
||||
expectedSetCost: 74,
|
||||
},
|
||||
{
|
||||
name: "index of array of number maps",
|
||||
schemaGenerator: getMapArrayWithRule("number", `self[0].x > 2.0`),
|
||||
expectedCalcCost: 4,
|
||||
setMaxElements: 3000,
|
||||
expectedSetCost: 4,
|
||||
},
|
||||
{
|
||||
name: "duration array with all",
|
||||
schemaGenerator: genArrayWithRule("duration", "self.all(x, true)"),
|
||||
expectedCalcCost: 2359295,
|
||||
setMaxElements: 5,
|
||||
expectedSetCost: 17,
|
||||
},
|
||||
{
|
||||
name: "index of duration array",
|
||||
schemaGenerator: genArrayWithRule("duration", "self[0].getHours() == 2"),
|
||||
expectedCalcCost: 4,
|
||||
setMaxElements: 525,
|
||||
expectedSetCost: 4,
|
||||
},
|
||||
{
|
||||
name: "date array with all",
|
||||
schemaGenerator: genArrayWithRule("date", "self.all(x, true)"),
|
||||
expectedCalcCost: 725936,
|
||||
setMaxElements: 15,
|
||||
expectedSetCost: 47,
|
||||
},
|
||||
{
|
||||
name: "index of date array",
|
||||
schemaGenerator: genArrayWithRule("date", "self[2].getDayOfMonth() == 13"),
|
||||
expectedCalcCost: 4,
|
||||
setMaxElements: 42,
|
||||
expectedSetCost: 4,
|
||||
},
|
||||
{
|
||||
name: "date-time array with all",
|
||||
schemaGenerator: genArrayWithRule("date-time", "self.all(x, true)"),
|
||||
expectedCalcCost: 428963,
|
||||
setMaxElements: 25,
|
||||
expectedSetCost: 77,
|
||||
},
|
||||
{
|
||||
name: "index of date-time array",
|
||||
schemaGenerator: genArrayWithRule("date-time", "self[2].getMinutes() == 45"),
|
||||
expectedCalcCost: 4,
|
||||
setMaxElements: 99,
|
||||
expectedSetCost: 4,
|
||||
},
|
||||
{
|
||||
name: "map of int-or-strings with all",
|
||||
schemaGenerator: genMapWithRule("int-or-string", "self.all(x, true)"),
|
||||
expectedCalcCost: 1348169,
|
||||
setMaxElements: 15,
|
||||
expectedSetCost: 47,
|
||||
},
|
||||
{
|
||||
name: "map of int-or-strings with has",
|
||||
schemaGenerator: genMapWithRule("int-or-string", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 5000,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of objects with all",
|
||||
schemaGenerator: genMapWithCustomItemRule(genObjectForMap(), "self.all(x, true)"),
|
||||
expectedCalcCost: 428963,
|
||||
setMaxElements: 20,
|
||||
expectedSetCost: 62,
|
||||
},
|
||||
{
|
||||
name: "map of objects with has",
|
||||
schemaGenerator: genMapWithCustomItemRule(genObjectForMap(), "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 9001,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of number maps with all",
|
||||
schemaGenerator: genMapWithCustomItemRule(genMapForMap(), "self.all(x, true)"),
|
||||
expectedCalcCost: 1179647,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 32,
|
||||
},
|
||||
{
|
||||
name: "map of number maps with has",
|
||||
schemaGenerator: genMapWithCustomItemRule(genMapForMap(), "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 101,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of number arrays with all",
|
||||
schemaGenerator: genMapWithCustomItemRule(genArrayForMap(), "self.all(x, true)"),
|
||||
expectedCalcCost: 1179647,
|
||||
setMaxElements: 25,
|
||||
expectedSetCost: 77,
|
||||
},
|
||||
{
|
||||
name: "map of number arrays with has",
|
||||
schemaGenerator: genMapWithCustomItemRule(genArrayForMap(), "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 40000,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of durations with all",
|
||||
schemaGenerator: genMapWithRule("duration", "self.all(x, true)"),
|
||||
expectedCalcCost: 1048577,
|
||||
setMaxElements: 5,
|
||||
expectedSetCost: 17,
|
||||
},
|
||||
{
|
||||
name: "map of durations with has",
|
||||
schemaGenerator: genMapWithRule("duration", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 256,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of dates with all",
|
||||
schemaGenerator: genMapWithRule("date", "self.all(x, true)"),
|
||||
expectedCalcCost: 524288,
|
||||
setMaxElements: 10,
|
||||
expectedSetCost: 32,
|
||||
},
|
||||
{
|
||||
name: "map of dates with has",
|
||||
schemaGenerator: genMapWithRule("date", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 65536,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
{
|
||||
name: "map of date-times with all",
|
||||
schemaGenerator: genMapWithRule("date-time", "self.all(x, true)"),
|
||||
expectedCalcCost: 349526,
|
||||
setMaxElements: 25,
|
||||
expectedSetCost: 77,
|
||||
},
|
||||
{
|
||||
name: "map of date-times with has",
|
||||
schemaGenerator: genMapWithRule("date-time", "has(self.x)"),
|
||||
expectedCalcCost: 0,
|
||||
setMaxElements: 490,
|
||||
expectedSetCost: 0,
|
||||
},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
// dynamic maxLength case
|
||||
schema := testCase.schemaGenerator(nil)
|
||||
t.Run("calc maxLength", schemaChecker(schema, testCase.expectedCalcCost, testCase.expectCalcCostExceedsLimit, t))
|
||||
// static maxLength case
|
||||
setSchema := testCase.schemaGenerator(&testCase.setMaxElements)
|
||||
t.Run("set maxLength", schemaChecker(setSchema, testCase.expectedSetCost, 0, t))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1682,12 +1682,17 @@ func TestValidationExpressions(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
for i := range tests {
|
||||
i := i
|
||||
t.Run(tests[i].name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// set costBudget to maxInt64 for current test
|
||||
tt := tests[i]
|
||||
tt.costBudget = math.MaxInt64
|
||||
for _, validRule := range tt.valid {
|
||||
for j := range tt.valid {
|
||||
validRule := tt.valid[j]
|
||||
t.Run(validRule, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := withRule(*tt.schema, validRule)
|
||||
celValidator := NewValidator(&s, PerCallLimit)
|
||||
if celValidator == nil {
|
||||
|
||||
@@ -15,9 +15,36 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/google/cel-go/checker/decls"
|
||||
"github.com/google/cel-go/common/types"
|
||||
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
// the largest request that will be accepted is 3MB
|
||||
// TODO(DangerOnTheRanger): wire in MaxRequestBodyBytes from apiserver/pkg/server/options/server_run_options.go to make this configurable
|
||||
maxRequestSizeBytes = int64(3 * 1024 * 1024)
|
||||
// OpenAPI duration strings follow RFC 3339, section 5.6 - see the comment on maxDatetimeSizeJSON
|
||||
maxDurationSizeJSON = 32
|
||||
// OpenAPI datetime strings follow RFC 3339, section 5.6, and the longest possible
|
||||
// such string is 9999-12-31T23:59:59.999999999Z, which has length 30 - we add 2
|
||||
// to allow for quotation marks
|
||||
maxDatetimeSizeJSON = 32
|
||||
// Golang allows a string of 0 to be parsed as a duration, so that plus 2 to account for
|
||||
// quotation marks makes 3
|
||||
minDurationSizeJSON = 3
|
||||
// RFC 3339 dates require YYYY-MM-DD, and then we add 2 to allow for quotation marks
|
||||
dateSizeJSON = 12
|
||||
// RFC 3339 times require 2-digit 24-hour time at the very least plus a capital T at the start,
|
||||
// e.g., T23, and we add 2 to allow for quotation marks as usual
|
||||
minTimeSizeJSON = 5
|
||||
// RFC 3339 datetimes require a full date (YYYY-MM-DD) and full time (HH:MM:SS), and we add 3 for
|
||||
// quotation marks like always in addition to the capital T that separates the date and time
|
||||
minDatetimeSizeJSON = 21
|
||||
)
|
||||
|
||||
// SchemaDeclType converts the structural schema to a CEL declaration, or returns nil if the
|
||||
// the structural schema should not be exposed in CEL expressions.
|
||||
// Set isResourceRoot to true for the root of a custom resource or embedded resource.
|
||||
@@ -44,7 +71,10 @@ func SchemaDeclType(s *schema.Structural, isResourceRoot bool) *DeclType {
|
||||
// To validate requirements on both the int and string representation:
|
||||
// `type(intOrStringField) == int ? intOrStringField < 5 : double(intOrStringField.replace('%', '')) < 0.5
|
||||
//
|
||||
return DynType
|
||||
dyn := newSimpleType("dyn", decls.Dyn, nil)
|
||||
// handle x-kubernetes-int-or-string by returning the max length of the largest possible string
|
||||
dyn.MaxElements = maxRequestSizeBytes - 2
|
||||
return dyn
|
||||
}
|
||||
|
||||
// We ignore XPreserveUnknownFields since we don't support validation rules on
|
||||
@@ -61,8 +91,14 @@ func SchemaDeclType(s *schema.Structural, isResourceRoot bool) *DeclType {
|
||||
case "array":
|
||||
if s.Items != nil {
|
||||
itemsType := SchemaDeclType(s.Items, s.Items.XEmbeddedResource)
|
||||
var maxItems int64
|
||||
if s.ValueValidation != nil && s.ValueValidation.MaxItems != nil {
|
||||
maxItems = *s.ValueValidation.MaxItems
|
||||
} else {
|
||||
maxItems = estimateMaxArrayItemsPerRequest(s.Items)
|
||||
}
|
||||
if itemsType != nil {
|
||||
return NewListType(itemsType)
|
||||
return NewListType(itemsType, maxItems)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -70,7 +106,13 @@ func SchemaDeclType(s *schema.Structural, isResourceRoot bool) *DeclType {
|
||||
if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil {
|
||||
propsType := SchemaDeclType(s.AdditionalProperties.Structural, s.AdditionalProperties.Structural.XEmbeddedResource)
|
||||
if propsType != nil {
|
||||
return NewMapType(StringType, propsType)
|
||||
var maxProperties int64
|
||||
if s.ValueValidation != nil && s.ValueValidation.MaxProperties != nil {
|
||||
maxProperties = *s.ValueValidation.MaxProperties
|
||||
} else {
|
||||
maxProperties = estimateMaxAdditionalPropertiesPerRequest(s.AdditionalProperties.Structural)
|
||||
}
|
||||
return NewMapType(StringType, propsType, maxProperties)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -106,14 +148,34 @@ func SchemaDeclType(s *schema.Structural, isResourceRoot bool) *DeclType {
|
||||
if s.ValueValidation != nil {
|
||||
switch s.ValueValidation.Format {
|
||||
case "byte":
|
||||
return BytesType
|
||||
byteWithMaxLength := newSimpleType("bytes", decls.Bytes, types.Bytes([]byte{}))
|
||||
if s.ValueValidation.MaxLength != nil {
|
||||
byteWithMaxLength.MaxElements = *s.ValueValidation.MaxLength
|
||||
} else {
|
||||
byteWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
|
||||
}
|
||||
return byteWithMaxLength
|
||||
case "duration":
|
||||
return DurationType
|
||||
durationWithMaxLength := newSimpleType("duration", decls.Duration, types.Duration{Duration: time.Duration(0)})
|
||||
durationWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
|
||||
return durationWithMaxLength
|
||||
case "date", "date-time":
|
||||
return TimestampType
|
||||
timestampWithMaxLength := newSimpleType("timestamp", decls.Timestamp, types.Timestamp{Time: time.Time{}})
|
||||
timestampWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
|
||||
return timestampWithMaxLength
|
||||
}
|
||||
}
|
||||
return StringType
|
||||
strWithMaxLength := newSimpleType("string", decls.String, types.String(""))
|
||||
if s.ValueValidation != nil && s.ValueValidation.MaxLength != nil {
|
||||
// multiply the user-provided max length by 4 in the case of an otherwise-untyped string
|
||||
// we do this because the OpenAPIv3 spec indicates that maxLength is specified in runes/code points,
|
||||
// but we need to reason about length for things like request size, so we use bytes in this code (and an individual
|
||||
// unicode code point can be up to 4 bytes long)
|
||||
strWithMaxLength.MaxElements = *s.ValueValidation.MaxLength * 4
|
||||
} else {
|
||||
strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
|
||||
}
|
||||
return strWithMaxLength
|
||||
case "boolean":
|
||||
return BoolType
|
||||
case "number":
|
||||
@@ -137,8 +199,8 @@ func WithTypeAndObjectMeta(s *schema.Structural) *schema.Structural {
|
||||
return s
|
||||
}
|
||||
result := &schema.Structural{
|
||||
Generic: s.Generic,
|
||||
Extensions: s.Extensions,
|
||||
Generic: s.Generic,
|
||||
Extensions: s.Extensions,
|
||||
ValueValidation: s.ValueValidation,
|
||||
}
|
||||
props := make(map[string]schema.Structural, len(s.Properties))
|
||||
@@ -151,7 +213,7 @@ func WithTypeAndObjectMeta(s *schema.Structural) *schema.Structural {
|
||||
props["metadata"] = schema.Structural{
|
||||
Generic: schema.Generic{Type: "object"},
|
||||
Properties: map[string]schema.Structural{
|
||||
"name": stringType,
|
||||
"name": stringType,
|
||||
"generateName": stringType,
|
||||
},
|
||||
}
|
||||
@@ -159,3 +221,99 @@ func WithTypeAndObjectMeta(s *schema.Structural) *schema.Structural {
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// estimateMinSizeJSON estimates the minimum size in bytes of the given schema when serialized in JSON.
|
||||
// minLength/minProperties/minItems are not currently taken into account, so if these limits are set the
|
||||
// minimum size might be higher than what estimateMinSizeJSON returns.
|
||||
func estimateMinSizeJSON(s *schema.Structural) int64 {
|
||||
if s == nil {
|
||||
// minimum valid JSON token has length 1 (single-digit number like `0`)
|
||||
return 1
|
||||
}
|
||||
switch s.Type {
|
||||
case "boolean":
|
||||
// true
|
||||
return 4
|
||||
case "number", "integer":
|
||||
// 0
|
||||
return 1
|
||||
case "string":
|
||||
if s.ValueValidation != nil {
|
||||
switch s.ValueValidation.Format {
|
||||
case "duration":
|
||||
return minDurationSizeJSON
|
||||
case "date":
|
||||
return dateSizeJSON
|
||||
case "date-time":
|
||||
return minDatetimeSizeJSON
|
||||
}
|
||||
}
|
||||
// ""
|
||||
return 2
|
||||
case "array":
|
||||
// []
|
||||
return 2
|
||||
case "object":
|
||||
// {}
|
||||
objSize := int64(2)
|
||||
// exclude optional fields since the request can omit them
|
||||
if s.ValueValidation != nil {
|
||||
for _, propName := range s.ValueValidation.Required {
|
||||
if prop, ok := s.Properties[propName]; ok {
|
||||
if prop.Default.Object != nil {
|
||||
// exclude fields with a default, those are filled in server-side
|
||||
continue
|
||||
}
|
||||
// add 4, 2 for quotations around the property name, 1 for the colon, and 1 for a comma
|
||||
objSize += int64(len(propName)) + estimateMinSizeJSON(&prop) + 4
|
||||
}
|
||||
}
|
||||
}
|
||||
return objSize
|
||||
}
|
||||
if s.XIntOrString {
|
||||
// 0
|
||||
return 1
|
||||
}
|
||||
// this code should be unreachable, so return the safest possible value considering this can be used as
|
||||
// a divisor
|
||||
return 1
|
||||
}
|
||||
|
||||
// estimateMaxArrayItemsPerRequest estimates the maximum number of array items with
|
||||
// the provided schema that can fit into a single request.
|
||||
func estimateMaxArrayItemsPerRequest(itemSchema *schema.Structural) int64 {
|
||||
// subtract 2 to account for [ and ]
|
||||
return (maxRequestSizeBytes - 2) / (estimateMinSizeJSON(itemSchema) + 1)
|
||||
}
|
||||
|
||||
// estimateMaxStringLengthPerRequest estimates the maximum string length (in characters)
|
||||
// of a string compatible with the format requirements in the provided schema.
|
||||
// must only be called on schemas of type "string" or x-kubernetes-int-or-string: true
|
||||
func estimateMaxStringLengthPerRequest(s *schema.Structural) int64 {
|
||||
if s.ValueValidation == nil || s.XIntOrString {
|
||||
// subtract 2 to account for ""
|
||||
return (maxRequestSizeBytes - 2)
|
||||
}
|
||||
switch s.ValueValidation.Format {
|
||||
case "duration":
|
||||
return maxDurationSizeJSON
|
||||
case "date":
|
||||
return dateSizeJSON
|
||||
case "date-time":
|
||||
return maxDatetimeSizeJSON
|
||||
default:
|
||||
// subtract 2 to account for ""
|
||||
return (maxRequestSizeBytes - 2)
|
||||
}
|
||||
}
|
||||
|
||||
// estimateMaxAdditionalPropertiesPerRequest estimates the maximum number of additional properties
|
||||
// with the provided schema that can fit into a single request.
|
||||
func estimateMaxAdditionalPropertiesPerRequest(additionalPropertiesSchema *schema.Structural) int64 {
|
||||
// 2 bytes for key + "" + colon + comma + smallest possible value, realistically the actual keys
|
||||
// will all vary in length
|
||||
keyValuePairSize := estimateMinSizeJSON(additionalPropertiesSchema) + 6
|
||||
// subtract 2 to account for { and }
|
||||
return (maxRequestSizeBytes - 2) / keyValuePairSize
|
||||
}
|
||||
|
||||
@@ -238,3 +238,267 @@ func testSchema() *schema.Structural {
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
func arraySchema(arrayType, format string, maxItems *int64) *schema.Structural {
|
||||
return &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: arrayType,
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: format,
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxItems: maxItems,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEstimateMaxLengthJSON(t *testing.T) {
|
||||
type maxLengthTest struct {
|
||||
Name string
|
||||
InputSchema *schema.Structural
|
||||
ExpectedMaxElements int64
|
||||
}
|
||||
tests := []maxLengthTest{
|
||||
{
|
||||
Name: "booleanArray",
|
||||
InputSchema: arraySchema("boolean", "", nil),
|
||||
// expected JSON is [true,true,...], so our length should be (maxRequestSizeBytes - 2) / 5
|
||||
ExpectedMaxElements: 629145,
|
||||
},
|
||||
{
|
||||
Name: "durationArray",
|
||||
InputSchema: arraySchema("string", "duration", nil),
|
||||
// expected JSON is ["0","0",...] so our length should be (maxRequestSizeBytes - 2) / 4
|
||||
ExpectedMaxElements: 786431,
|
||||
},
|
||||
{
|
||||
Name: "datetimeArray",
|
||||
InputSchema: arraySchema("string", "date-time", nil),
|
||||
// expected JSON is ["2000-01-01T01:01:01","2000-01-01T01:01:01",...] so our length should be (maxRequestSizeBytes - 2) / 22
|
||||
ExpectedMaxElements: 142987,
|
||||
},
|
||||
{
|
||||
Name: "dateArray",
|
||||
InputSchema: arraySchema("string", "date", nil),
|
||||
// expected JSON is ["2000-01-01","2000-01-02",...] so our length should be (maxRequestSizeBytes - 2) / 13
|
||||
ExpectedMaxElements: 241978,
|
||||
},
|
||||
{
|
||||
Name: "numberArray",
|
||||
InputSchema: arraySchema("integer", "", nil),
|
||||
// expected JSON is [0,0,...] so our length should be (maxRequestSizeBytes - 2) / 2
|
||||
ExpectedMaxElements: 1572863,
|
||||
},
|
||||
{
|
||||
Name: "stringArray",
|
||||
InputSchema: arraySchema("string", "", nil),
|
||||
// expected JSON is ["","",...] so our length should be (maxRequestSizeBytes - 2) / 3
|
||||
ExpectedMaxElements: 1048575,
|
||||
},
|
||||
{
|
||||
Name: "stringMap",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
// expected JSON is {"":"","":"",...} so our length should be (3000000 - 2) / 6
|
||||
ExpectedMaxElements: 393215,
|
||||
},
|
||||
{
|
||||
Name: "objectOptionalPropertyArray",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"required": schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
"optional": schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"required"},
|
||||
},
|
||||
},
|
||||
},
|
||||
// expected JSON is [{"required":"",},{"required":"",},...] so our length should be (maxRequestSizeBytes - 2) / 17
|
||||
ExpectedMaxElements: 185042,
|
||||
},
|
||||
{
|
||||
Name: "arrayWithLength",
|
||||
InputSchema: arraySchema("integer", "int64", maxPtr(10)),
|
||||
// manually set by MaxItems
|
||||
ExpectedMaxElements: 10,
|
||||
},
|
||||
{
|
||||
Name: "stringWithLength",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
MaxLength: maxPtr(20),
|
||||
},
|
||||
},
|
||||
// manually set by MaxLength, but we expect a 4x multiplier compared to the original input
|
||||
// since OpenAPIv3 maxLength uses code points, but DeclType works with bytes
|
||||
ExpectedMaxElements: 80,
|
||||
},
|
||||
{
|
||||
Name: "mapWithLength",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
AdditionalProperties: &schema.StructuralOrBool{Structural: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
}},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "string",
|
||||
MaxProperties: maxPtr(15),
|
||||
},
|
||||
},
|
||||
// manually set by MaxProperties
|
||||
ExpectedMaxElements: 15,
|
||||
},
|
||||
{
|
||||
Name: "durationMaxSize",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "duration",
|
||||
},
|
||||
},
|
||||
// should be exactly equal to maxDurationSizeJSON
|
||||
ExpectedMaxElements: maxDurationSizeJSON,
|
||||
},
|
||||
{
|
||||
Name: "dateSize",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "date",
|
||||
},
|
||||
},
|
||||
// should be exactly equal to dateSizeJSON
|
||||
ExpectedMaxElements: dateSizeJSON,
|
||||
},
|
||||
{
|
||||
Name: "maxdatetimeSize",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "date-time",
|
||||
},
|
||||
},
|
||||
// should be exactly equal to maxDatetimeSizeJSON
|
||||
ExpectedMaxElements: maxDatetimeSizeJSON,
|
||||
},
|
||||
{
|
||||
Name: "maxintOrStringSize",
|
||||
InputSchema: &schema.Structural{
|
||||
Extensions: schema.Extensions{
|
||||
XIntOrString: true,
|
||||
},
|
||||
},
|
||||
// should be exactly equal to maxRequestSizeBytes - 2 (to allow for quotes in the case of a string)
|
||||
ExpectedMaxElements: maxRequestSizeBytes - 2,
|
||||
},
|
||||
{
|
||||
Name: "objectDefaultFieldArray",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "array",
|
||||
},
|
||||
Items: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "object",
|
||||
},
|
||||
Properties: map[string]schema.Structural{
|
||||
"field": schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
Default: schema.JSON{Object: "default"},
|
||||
},
|
||||
},
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Required: []string{"field"},
|
||||
},
|
||||
},
|
||||
},
|
||||
// expected JSON is [{},{},...] so our length should be (maxRequestSizeBytes - 2) / 3
|
||||
ExpectedMaxElements: 1048575,
|
||||
},
|
||||
{
|
||||
Name: "byteStringSize",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "byte",
|
||||
},
|
||||
},
|
||||
// expected JSON is "" so our length should be (maxRequestSizeBytes - 2)
|
||||
ExpectedMaxElements: 3145726,
|
||||
},
|
||||
{
|
||||
Name: "byteStringSetMaxLength",
|
||||
InputSchema: &schema.Structural{
|
||||
Generic: schema.Generic{
|
||||
Type: "string",
|
||||
},
|
||||
ValueValidation: &schema.ValueValidation{
|
||||
Format: "byte",
|
||||
MaxLength: maxPtr(20),
|
||||
},
|
||||
},
|
||||
// note that unlike regular strings we don't have to take unicode into account,
|
||||
// so we we expect the max length to be exactly equal to the user-supplied one
|
||||
ExpectedMaxElements: 20,
|
||||
},
|
||||
}
|
||||
for _, testCase := range tests {
|
||||
t.Run(testCase.Name, func(t *testing.T) {
|
||||
decl := SchemaDeclType(testCase.InputSchema, false)
|
||||
if decl.MaxElements != testCase.ExpectedMaxElements {
|
||||
t.Errorf("wrong maxElements (got %d, expected %d)", decl.MaxElements, testCase.ExpectedMaxElements)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func maxPtr(max int64) *int64 {
|
||||
return &max
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ package model
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/google/cel-go/cel"
|
||||
@@ -30,22 +31,28 @@ import (
|
||||
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
noMaxLength = math.MaxInt
|
||||
)
|
||||
|
||||
// NewListType returns a parameterized list type with a specified element type.
|
||||
func NewListType(elem *DeclType) *DeclType {
|
||||
func NewListType(elem *DeclType, maxItems int64) *DeclType {
|
||||
return &DeclType{
|
||||
name: "list",
|
||||
ElemType: elem,
|
||||
MaxElements: maxItems,
|
||||
exprType: decls.NewListType(elem.ExprType()),
|
||||
defaultValue: NewListValue(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewMapType returns a parameterized map type with the given key and element types.
|
||||
func NewMapType(key, elem *DeclType) *DeclType {
|
||||
func NewMapType(key, elem *DeclType, maxProperties int64) *DeclType {
|
||||
return &DeclType{
|
||||
name: "map",
|
||||
KeyType: key,
|
||||
ElemType: elem,
|
||||
MaxElements: maxProperties,
|
||||
exprType: decls.NewMapType(key.ExprType(), elem.ExprType()),
|
||||
defaultValue: NewMapValue(),
|
||||
}
|
||||
@@ -97,12 +104,13 @@ func newSimpleType(name string, exprType *exprpb.Type, zeroVal ref.Val) *DeclTyp
|
||||
type DeclType struct {
|
||||
fmt.Stringer
|
||||
|
||||
name string
|
||||
Fields map[string]*DeclField
|
||||
KeyType *DeclType
|
||||
ElemType *DeclType
|
||||
TypeParam bool
|
||||
Metadata map[string]string
|
||||
name string
|
||||
Fields map[string]*DeclField
|
||||
KeyType *DeclType
|
||||
ElemType *DeclType
|
||||
TypeParam bool
|
||||
Metadata map[string]string
|
||||
MaxElements int64
|
||||
|
||||
exprType *exprpb.Type
|
||||
traitMask int
|
||||
@@ -160,7 +168,7 @@ func (t *DeclType) MaybeAssignTypeName(name string) *DeclType {
|
||||
if updated == t.ElemType {
|
||||
return t
|
||||
}
|
||||
return NewMapType(t.KeyType, updated)
|
||||
return NewMapType(t.KeyType, updated, t.MaxElements)
|
||||
}
|
||||
if t.IsList() {
|
||||
elemTypeName := fmt.Sprintf("%s.@idx", name)
|
||||
@@ -168,7 +176,7 @@ func (t *DeclType) MaybeAssignTypeName(name string) *DeclType {
|
||||
if updated == t.ElemType {
|
||||
return t
|
||||
}
|
||||
return NewListType(updated)
|
||||
return NewListType(updated, t.MaxElements)
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -547,8 +555,8 @@ var (
|
||||
UintType = newSimpleType("uint", decls.Uint, types.Uint(0))
|
||||
|
||||
// ListType is equivalent to the CEL 'list' type.
|
||||
ListType = NewListType(AnyType)
|
||||
ListType = NewListType(AnyType, noMaxLength)
|
||||
|
||||
// MapType is equivalent to the CEL 'map' type.
|
||||
MapType = NewMapType(AnyType, AnyType)
|
||||
MapType = NewMapType(AnyType, AnyType, noMaxLength)
|
||||
)
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
)
|
||||
|
||||
func TestTypes_ListType(t *testing.T) {
|
||||
list := NewListType(StringType)
|
||||
list := NewListType(StringType, -1)
|
||||
if !list.IsList() {
|
||||
t.Error("list type not identifiable as list")
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func TestTypes_ListType(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTypes_MapType(t *testing.T) {
|
||||
mp := NewMapType(StringType, IntType)
|
||||
mp := NewMapType(StringType, IntType, -1)
|
||||
if !mp.IsMap() {
|
||||
t.Error("map type not identifiable as map")
|
||||
}
|
||||
|
||||
@@ -359,6 +359,9 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
|
||||
// A request body might be encoded in json, and is converted to
|
||||
// proto when persisted in etcd, so we allow 2x as the largest request
|
||||
// body size to be accepted and decoded in a write request.
|
||||
// If this constant is changed, maxRequestSizeBytes in apiextensions-apiserver/third_party/forked/celopenapi/model/schemas.go
|
||||
// should be changed to reflect the new value, if the two haven't
|
||||
// been wired together already somehow.
|
||||
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
|
||||
|
||||
// Default to treating watch as a long-running operation
|
||||
|
||||
Reference in New Issue
Block a user