github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Prioritizing nodes based on volume capacity: API changes

Browse Source
This commit is contained in:
Yecheng Fu
2021-06-20 10:00:51 +08:00
parent 7ad7c0757a
commit b522e95aae
17 changed files with 426 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
pkg/scheduler/apis/config/scheme/scheme_test.go
View File

@@ -733,6 +733,16 @@ func TestCodecsEncodePluginConfig(t *testing.T) {
Args: runtime.RawExtension{
Object: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(300),
Shape: []v1beta1.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
},
@@ -804,6 +814,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta1
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
@@ -855,6 +870,16 @@ profiles:
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
Shape: []config.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
{
@@ -913,6 +938,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta1
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
@@ -945,6 +975,16 @@ profiles:
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(300),
Shape: []v1beta2.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
},
@@ -1009,6 +1049,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta2
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta2

15
pkg/scheduler/apis/config/types_pluginargs.go
View File

@@ -214,6 +214,21 @@ type VolumeBindingArgs struct {
// Value must be non-negative integer. The value zero indicates no waiting.
// If this value is nil, the default value will be used.
BindTimeoutSeconds int64
// Shape specifies the points defining the score function shape, which is
// used to score nodes based on the utilization of statically provisioned
// PVs. The utilization is calculated by dividing the total requested
// storage of the pod by the total capacity of feasible PVs on each node.
// Each point contains utilization (ranges from 0 to 100) and its
// associated score (ranges from 0 to 10). You can turn the priority by
// specifying different scores for different utilization numbers.
// The default shape points are:
// 1) 0 for 0 utilization
// 2) 10 for 100 utilization
// All points must be sorted in increasing order by utilization.
// +featureGate=VolumeCapacityPriority
// +optional
Shape []UtilizationShapePoint
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

12
pkg/scheduler/apis/config/v1beta1/defaults.go
View File

@@ -280,6 +280,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta1.VolumeBindingArgs) {
if obj.BindTimeoutSeconds == nil {
obj.BindTimeoutSeconds = pointer.Int64Ptr(600)
}
if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
obj.Shape = []v1beta1.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta1.PodTopologySpreadArgs) {

24
pkg/scheduler/apis/config/v1beta1/defaults_test.go
View File

@@ -700,6 +700,30 @@ func TestPluginArgsDefaults(t *testing.T) {
},
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
in: &v1beta1.VolumeBindingArgs{},
want: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
in: &v1beta1.VolumeBindingArgs{},
want: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
Shape: []v1beta1.UtilizationShapePoint{
{Utilization: 0, Score: 0},
{Utilization: 100, Score: 10},
},
},
},
}
for _, tc := range tests {
scheme := runtime.NewScheme()

2
pkg/scheduler/apis/config/v1beta1/zz_generated.conversion.go generated
View File

@@ -916,6 +916,7 @@ func autoConvert_v1beta1_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet
if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}
@@ -928,6 +929,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta1_VolumeBindingArgs(in *confi
if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]v1beta1.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}

12
pkg/scheduler/apis/config/v1beta2/defaults.go
View File

@@ -245,6 +245,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta2.VolumeBindingArgs) {
if obj.BindTimeoutSeconds == nil {
obj.BindTimeoutSeconds = pointer.Int64Ptr(600)
}
if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
obj.Shape = []v1beta2.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta2.PodTopologySpreadArgs) {

24
pkg/scheduler/apis/config/v1beta2/defaults_test.go
View File

@@ -674,6 +674,30 @@ func TestPluginArgsDefaults(t *testing.T) {
},
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
in: &v1beta2.VolumeBindingArgs{},
want: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
in: &v1beta2.VolumeBindingArgs{},
want: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
Shape: []v1beta2.UtilizationShapePoint{
{Utilization: 0, Score: 0},
{Utilization: 100, Score: 10},
},
},
},
}
for _, tc := range tests {
scheme := runtime.NewScheme()

2
pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go generated
View File

@@ -815,6 +815,7 @@ func autoConvert_v1beta2_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet
if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}
@@ -827,6 +828,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta2_VolumeBindingArgs(in *confi
if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]v1beta2.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}

16
pkg/scheduler/apis/config/validation/validation_pluginargs.go
View File

@@ -25,7 +25,9 @@ import (
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
)
@@ -294,13 +296,21 @@ func ValidateNodeAffinityArgs(path *field.Path, args *config.NodeAffinityArgs) e
// ValidateVolumeBindingArgs validates that VolumeBindingArgs are set correctly.
func ValidateVolumeBindingArgs(path *field.Path, args *config.VolumeBindingArgs) error {
var err error
var allErrs field.ErrorList
if args.BindTimeoutSeconds < 0 {
err = field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value")
allErrs = append(allErrs, field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value"))
}
return err
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
allErrs = append(allErrs, validateFunctionShape(args.Shape, path.Child("shape"))...)
} else if args.Shape != nil {
// When the feature is off, return an error if the config is not nil.
// This prevents unexpected configuration from taking effect when the
// feature turns on in the future.
allErrs = append(allErrs, field.Invalid(path.Child("shape"), args.Shape, "unexpected field `shape`, remove it or turn on the feature gate VolumeCapacityPriority"))
}
return allErrs.ToAggregate()
}
func ValidateNodeResourcesFitArgs(path *field.Path, args *config.NodeResourcesFitArgs) error {

116
pkg/scheduler/apis/config/validation/validation_pluginargs_test.go
View File

@@ -25,7 +25,12 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
)
@@ -950,9 +955,10 @@ func TestValidateNodeAffinityArgs(t *testing.T) {
func TestValidateVolumeBindingArgs(t *testing.T) {
cases := []struct {
name string
args config.VolumeBindingArgs
wantErr error
name string
args config.VolumeBindingArgs
features map[featuregate.Feature]bool
wantErr error
}{
{
name: "zero is a valid config",
@@ -971,14 +977,112 @@ func TestValidateVolumeBindingArgs(t *testing.T) {
args: config.VolumeBindingArgs{
BindTimeoutSeconds: -10,
},
wantErr: &field.Error{
Type: field.ErrorTypeInvalid,
Field: "bindTimeoutSeconds",
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "bindTimeoutSeconds",
BadValue: int64(-10),
Detail: "invalid BindTimeoutSeconds, should not be a negative value",
}}),
},
{
name: "[VolumeCapacityPriority=off] shape should be nil when the feature is off",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: nil,
},
},
{
name: "[VolumeCapacityPriority=off] error if the shape is not nil when the feature is off",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: 1, Score: 1},
{Utilization: 3, Score: 3},
},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape should not be empty",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeRequired,
Field: "shape",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape points must be sorted in increasing order",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: 3, Score: 3},
{Utilization: 1, Score: 1},
},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[1].utilization",
Detail: "Invalid value: 1: utilization values must be sorted in increasing order",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape point: invalid utilization and score",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: -1, Score: 1},
{Utilization: 10, Score: -1},
{Utilization: 20, Score: 11},
{Utilization: 101, Score: 1},
},
},
wantErr: errors.NewAggregate([]error{
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[0].utilization",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[1].score",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[2].score",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[3].utilization",
},
}),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
for k, v := range tc.features {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
}
err := ValidateVolumeBindingArgs(nil, &tc.args)
if diff := cmp.Diff(tc.wantErr, err, ignoreBadValueDetail); diff != "" {
t.Errorf("ValidateVolumeBindingArgs returned err (-want,+got):\n%s", diff)

5
pkg/scheduler/apis/config/zz_generated.deepcopy.go generated
View File

@@ -970,6 +970,11 @@ func (in *UtilizationShapePoint) DeepCopy() *UtilizationShapePoint {
func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.Shape != nil {
in, out := &in.Shape, &out.Shape
*out = make([]UtilizationShapePoint, len(*in))
copy(*out, *in)
}
return
}