API changes for Topology aware dynamic provisioning

This commit is contained in:
lichuqiang
2018-05-23 11:24:31 +08:00
parent 50178d3ed3
commit 9a77899ee5
12 changed files with 523 additions and 18 deletions

View File

@@ -2166,6 +2166,7 @@ type NodeSelector struct {
// A null or empty node selector term matches no objects. The requirements of
// them are ANDed.
// The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
type NodeSelectorTerm struct {
// A list of node selector requirements by node's labels.
MatchExpressions []NodeSelectorRequirement
@@ -2203,6 +2204,27 @@ const (
NodeSelectorOpLt NodeSelectorOperator = "Lt"
)
// A topology selector term represents the result of label queries.
// A null or empty topology selector term matches no objects.
// The requirements of them are ANDed.
// It provides a subset of functionality as NodeSelectorTerm.
// This is an alpha feature and may change in the future.
type TopologySelectorTerm struct {
// A list of topology selector requirements by labels.
// +optional
MatchLabelExpressions []TopologySelectorLabelRequirement
}
// A topology selector requirement is a selector that matches given label.
// This is an alpha feature and may change in the future.
type TopologySelectorLabelRequirement struct {
// The label key that the selector applies to.
Key string
// An array of string values. One value must match the label to be selected.
// Each entry in Values is ORed.
Values []string
}
// Affinity is a group of affinity scheduling rules.
type Affinity struct {
// Describes node affinity scheduling rules for the pod.

View File

@@ -316,6 +316,50 @@ func MatchNodeSelectorTerms(
return false
}
// TopologySelectorRequirementsAsSelector converts the []TopologySelectorLabelRequirement api type into a struct
// that implements labels.Selector.
func TopologySelectorRequirementsAsSelector(tsm []v1.TopologySelectorLabelRequirement) (labels.Selector, error) {
if len(tsm) == 0 {
return labels.Nothing(), nil
}
selector := labels.NewSelector()
for _, expr := range tsm {
r, err := labels.NewRequirement(expr.Key, selection.In, expr.Values)
if err != nil {
return nil, err
}
selector = selector.Add(*r)
}
return selector, nil
}
// MatchTopologySelectorTerms checks whether given labels match topology selector terms in ORed;
// nil or empty term matches no objects; while empty term list matches all objects.
func MatchTopologySelectorTerms(topologySelectorTerms []v1.TopologySelectorTerm, lbls labels.Set) bool {
if len(topologySelectorTerms) == 0 {
// empty term list matches all objects
return true
}
for _, req := range topologySelectorTerms {
// nil or empty term selects no objects
if len(req.MatchLabelExpressions) == 0 {
continue
}
labelSelector, err := TopologySelectorRequirementsAsSelector(req.MatchLabelExpressions)
if err != nil || !labelSelector.Matches(lbls) {
continue
}
return true
}
return false
}
// AddOrUpdateTolerationInPodSpec tries to add a toleration to the toleration list in PodSpec.
// Returns true if something was updated, false otherwise.
func AddOrUpdateTolerationInPodSpec(spec *v1.PodSpec, toleration *v1.Toleration) bool {

View File

@@ -301,6 +301,71 @@ func TestNodeSelectorRequirementsAsSelector(t *testing.T) {
}
}
func TestTopologySelectorRequirementsAsSelector(t *testing.T) {
mustParse := func(s string) labels.Selector {
out, e := labels.Parse(s)
if e != nil {
panic(e)
}
return out
}
tc := []struct {
in []v1.TopologySelectorLabelRequirement
out labels.Selector
expectErr bool
}{
{in: nil, out: labels.Nothing()},
{in: []v1.TopologySelectorLabelRequirement{}, out: labels.Nothing()},
{
in: []v1.TopologySelectorLabelRequirement{{
Key: "foo",
Values: []string{"bar", "baz"},
}},
out: mustParse("foo in (baz,bar)"),
},
{
in: []v1.TopologySelectorLabelRequirement{{
Key: "foo",
Values: []string{},
}},
expectErr: true,
},
{
in: []v1.TopologySelectorLabelRequirement{
{
Key: "foo",
Values: []string{"bar", "baz"},
},
{
Key: "invalid",
Values: []string{},
},
},
expectErr: true,
},
{
in: []v1.TopologySelectorLabelRequirement{{
Key: "/invalidkey",
Values: []string{"bar", "baz"},
}},
expectErr: true,
},
}
for i, tc := range tc {
out, err := TopologySelectorRequirementsAsSelector(tc.in)
if err == nil && tc.expectErr {
t.Errorf("[%v]expected error but got none.", i)
}
if err != nil && !tc.expectErr {
t.Errorf("[%v]did not expect error but got: %v", i, err)
}
if !reflect.DeepEqual(out, tc.out) {
t.Errorf("[%v]expected:\n\t%+v\nbut got:\n\t%+v", i, tc.out, out)
}
}
}
func TestTolerationsTolerateTaintsWithFilter(t *testing.T) {
testCases := []struct {
description string
@@ -792,3 +857,161 @@ func TestMatchNodeSelectorTerms(t *testing.T) {
})
}
}
func TestMatchTopologySelectorTerms(t *testing.T) {
type args struct {
topologySelectorTerms []v1.TopologySelectorTerm
labels labels.Set
}
tests := []struct {
name string
args args
want bool
}{
{
name: "nil term list",
args: args{
topologySelectorTerms: nil,
labels: nil,
},
want: true,
},
{
name: "nil term",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{},
},
},
labels: nil,
},
want: false,
},
{
name: "label matches MatchLabelExpressions terms",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{
Key: "label_1",
Values: []string{"label_1_val"},
}},
},
},
labels: map[string]string{"label_1": "label_1_val"},
},
want: true,
},
{
name: "label does not match MatchLabelExpressions terms",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{
Key: "label_1",
Values: []string{"label_1_val"},
}},
},
},
labels: map[string]string{"label_1": "label_1_val-failed"},
},
want: false,
},
{
name: "multi-values in one requirement, one matched",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{
Key: "label_1",
Values: []string{"label_1_val1", "label_1_val2"},
}},
},
},
labels: map[string]string{"label_1": "label_1_val2"},
},
want: true,
},
{
name: "multi-terms was set, one matched",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{
Key: "label_1",
Values: []string{"label_1_val"},
}},
},
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{{
Key: "label_2",
Values: []string{"label_2_val"},
}},
},
},
labels: map[string]string{
"label_2": "label_2_val",
},
},
want: true,
},
{
name: "multi-requirement in one term, fully matched",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{
{
Key: "label_1",
Values: []string{"label_1_val"},
},
{
Key: "label_2",
Values: []string{"label_2_val"},
},
},
},
},
labels: map[string]string{
"label_1": "label_1_val",
"label_2": "label_2_val",
},
},
want: true,
},
{
name: "multi-requirement in one term, partial matched",
args: args{
topologySelectorTerms: []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{
{
Key: "label_1",
Values: []string{"label_1_val"},
},
{
Key: "label_2",
Values: []string{"label_2_val"},
},
},
},
},
labels: map[string]string{
"label_1": "label_1_val-failed",
"label_2": "label_2_val",
},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := MatchTopologySelectorTerms(tt.args.topologySelectorTerms, tt.args.labels); got != tt.want {
t.Errorf("MatchTopologySelectorTermsORed() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -3104,6 +3104,32 @@ func ValidateNodeSelector(nodeSelector *core.NodeSelector, fldPath *field.Path)
return allErrs
}
// validateTopologySelectorLabelRequirement tests that the specified TopologySelectorLabelRequirement fields has valid data
func validateTopologySelectorLabelRequirement(rq core.TopologySelectorLabelRequirement, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if len(rq.Values) == 0 {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("values"), "must specify as least one value"))
}
allErrs = append(allErrs, unversionedvalidation.ValidateLabelName(rq.Key, fldPath.Child("key"))...)
return allErrs
}
// ValidateTopologySelectorTerm tests that the specified topology selector term has valid data
func ValidateTopologySelectorTerm(term core.TopologySelectorTerm, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
for i, req := range term.MatchLabelExpressions {
allErrs = append(allErrs, validateTopologySelectorLabelRequirement(req, fldPath.Child("matchLabelExpressions").Index(i))...)
}
} else if len(term.MatchLabelExpressions) != 0 {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling"))
}
return allErrs
}
// ValidateAvoidPodsInNodeAnnotations tests that the serialized AvoidPods in Node.Annotations has valid data
func ValidateAvoidPodsInNodeAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}

View File

@@ -72,6 +72,14 @@ type StorageClass struct {
// the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// +optional
AllowedTopologies []api.TopologySelectorTerm
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

View File

@@ -27,4 +27,7 @@ func DropDisabledAlphaFields(class *storage.StorageClass) {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
class.VolumeBindingMode = nil
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
class.AllowedTopologies = nil
}
}

View File

@@ -17,39 +17,60 @@ limitations under the License.
package util
import (
"reflect"
"testing"
utilfeature "k8s.io/apiserver/pkg/util/feature"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/storage"
)
func TestDropAlphaFields(t *testing.T) {
bindingMode := storage.VolumeBindingWaitForFirstConsumer
allowedTopologies := []api.TopologySelectorTerm{
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "kubernetes.io/hostname",
Values: []string{"node1"},
},
},
},
}
// Test that field gets dropped when feature gate is not set
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err)
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,DynamicProvisioningScheduling=false"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
}
class := &storage.StorageClass{
VolumeBindingMode: &bindingMode,
AllowedTopologies: allowedTopologies,
}
DropDisabledAlphaFields(class)
if class.VolumeBindingMode != nil {
t.Errorf("VolumeBindingMode field didn't get dropped: %+v", class.VolumeBindingMode)
}
if class.AllowedTopologies != nil {
t.Errorf("AllowedTopologies field didn't get dropped: %+v", class.AllowedTopologies)
}
// Test that field does not get dropped when feature gate is set
class = &storage.StorageClass{
VolumeBindingMode: &bindingMode,
AllowedTopologies: allowedTopologies,
}
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling: %v", err)
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=true,DynamicProvisioningScheduling=true"); err != nil {
t.Fatalf("Failed to set feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
}
DropDisabledAlphaFields(class)
if class.VolumeBindingMode != &bindingMode {
t.Errorf("VolumeBindingMode field got unexpectantly modified: %+v", class.VolumeBindingMode)
}
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false"); err != nil {
t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err)
if !reflect.DeepEqual(class.AllowedTopologies, allowedTopologies) {
t.Errorf("AllowedTopologies field got unexpectantly modified: %+v", class.AllowedTopologies)
}
if err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false,DynamicProvisioningScheduling=false"); err != nil {
t.Fatalf("Failed to disable feature gate for VolumeScheduling or DynamicProvisioningScheduling: %v", err)
}
}

View File

@@ -47,6 +47,7 @@ func ValidateStorageClass(storageClass *storage.StorageClass) field.ErrorList {
allErrs = append(allErrs, validateReclaimPolicy(storageClass.ReclaimPolicy, field.NewPath("reclaimPolicy"))...)
allErrs = append(allErrs, validateAllowVolumeExpansion(storageClass.AllowVolumeExpansion, field.NewPath("allowVolumeExpansion"))...)
allErrs = append(allErrs, validateVolumeBindingMode(storageClass.VolumeBindingMode, field.NewPath("volumeBindingMode"))...)
allErrs = append(allErrs, validateAllowedTopologies(storageClass.AllowedTopologies, field.NewPath("allowedTopologies"))...)
return allErrs
}
@@ -239,3 +240,22 @@ func validateVolumeBindingMode(mode *storage.VolumeBindingMode, fldPath *field.P
return allErrs
}
// validateAllowedTopology tests that AllowedTopologies specifies valid values.
func validateAllowedTopologies(topologies []api.TopologySelectorTerm, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if topologies == nil || len(topologies) == 0 {
return allErrs
}
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicProvisioningScheduling) {
allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate DynamicProvisioningScheduling"))
}
for i, term := range topologies {
allErrs = append(allErrs, apivalidation.ValidateTopologySelectorTerm(term, fldPath.Index(i))...)
}
return allErrs
}

View File

@@ -450,21 +450,22 @@ func TestVolumeAttachmentUpdateValidation(t *testing.T) {
}
}
func makeClassWithBinding(mode *storage.VolumeBindingMode) *storage.StorageClass {
func makeClass(mode *storage.VolumeBindingMode, topologies []api.TopologySelectorTerm) *storage.StorageClass {
return &storage.StorageClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "foo"},
Provisioner: "kubernetes.io/foo-provisioner",
ReclaimPolicy: &deleteReclaimPolicy,
VolumeBindingMode: mode,
AllowedTopologies: topologies,
}
}
// TODO: Remove these tests once feature gate is not required
func TestValidateVolumeBindingModeAlphaDisabled(t *testing.T) {
errorCases := map[string]*storage.StorageClass{
"immediate mode": makeClassWithBinding(&immediateMode1),
"waiting mode": makeClassWithBinding(&waitingMode),
"invalid mode": makeClassWithBinding(&invalidMode),
"immediate mode": makeClass(&immediateMode1, nil),
"waiting mode": makeClass(&waitingMode, nil),
"invalid mode": makeClass(&invalidMode, nil),
}
err := utilfeature.DefaultFeatureGate.Set("VolumeScheduling=false")
@@ -486,19 +487,19 @@ type bindingTest struct {
func TestValidateVolumeBindingMode(t *testing.T) {
cases := map[string]bindingTest{
"no mode": {
class: makeClassWithBinding(nil),
class: makeClass(nil, nil),
shouldSucceed: false,
},
"immediate mode": {
class: makeClassWithBinding(&immediateMode1),
class: makeClass(&immediateMode1, nil),
shouldSucceed: true,
},
"waiting mode": {
class: makeClassWithBinding(&waitingMode),
class: makeClass(&waitingMode, nil),
shouldSucceed: true,
},
"invalid mode": {
class: makeClassWithBinding(&invalidMode),
class: makeClass(&invalidMode, nil),
shouldSucceed: false,
},
}
@@ -532,10 +533,10 @@ type updateTest struct {
}
func TestValidateUpdateVolumeBindingMode(t *testing.T) {
noBinding := makeClassWithBinding(nil)
immediateBinding1 := makeClassWithBinding(&immediateMode1)
immediateBinding2 := makeClassWithBinding(&immediateMode2)
waitBinding := makeClassWithBinding(&waitingMode)
noBinding := makeClass(nil, nil)
immediateBinding1 := makeClass(&immediateMode1, nil)
immediateBinding2 := makeClass(&immediateMode2, nil)
waitBinding := makeClass(&waitingMode, nil)
cases := map[string]updateTest{
"old and new no mode": {
@@ -591,3 +592,102 @@ func TestValidateUpdateVolumeBindingMode(t *testing.T) {
t.Fatalf("Failed to disable feature gate for VolumeScheduling: %v", err)
}
}
func TestValidateAllowedTopologies(t *testing.T) {
validTopology := []api.TopologySelectorTerm{
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "failure-domain.beta.kubernetes.io/zone",
Values: []string{"zone1"},
},
{
Key: "kubernetes.io/hostname",
Values: []string{"node1"},
},
},
},
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "failure-domain.beta.kubernetes.io/zone",
Values: []string{"zone2"},
},
{
Key: "kubernetes.io/hostname",
Values: []string{"node2"},
},
},
},
}
topologyInvalidKey := []api.TopologySelectorTerm{
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "/invalidkey",
Values: []string{"zone1"},
},
},
},
}
topologyLackOfValues := []api.TopologySelectorTerm{
{
MatchLabelExpressions: []api.TopologySelectorLabelRequirement{
{
Key: "kubernetes.io/hostname",
Values: []string{},
},
},
},
}
cases := map[string]bindingTest{
"no topology": {
class: makeClass(nil, nil),
shouldSucceed: true,
},
"valid topology": {
class: makeClass(nil, validTopology),
shouldSucceed: true,
},
"topology invalid key": {
class: makeClass(nil, topologyInvalidKey),
shouldSucceed: false,
},
"topology lack of values": {
class: makeClass(nil, topologyLackOfValues),
shouldSucceed: false,
},
}
// TODO: remove when feature gate not required
err := utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=true")
if err != nil {
t.Fatalf("Failed to enable feature gate for DynamicProvisioningScheduling: %v", err)
}
for testName, testCase := range cases {
errs := ValidateStorageClass(testCase.class)
if testCase.shouldSucceed && len(errs) != 0 {
t.Errorf("Expected success for test %q, got %v", testName, errs)
}
if !testCase.shouldSucceed && len(errs) == 0 {
t.Errorf("Expected failure for test %q, got success", testName)
}
}
err = utilfeature.DefaultFeatureGate.Set("DynamicProvisioningScheduling=false")
if err != nil {
t.Fatalf("Failed to disable feature gate for DynamicProvisioningScheduling: %v", err)
}
for testName, testCase := range cases {
errs := ValidateStorageClass(testCase.class)
if len(errs) == 0 && testCase.class.AllowedTopologies != nil {
t.Errorf("Expected failure for test %q, got success", testName)
}
}
}

View File

@@ -2389,6 +2389,7 @@ type NodeSelector struct {
// A null or empty node selector term matches no objects. The requirements of
// them are ANDed.
// The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
type NodeSelectorTerm struct {
// A list of node selector requirements by node's labels.
// +optional
@@ -2428,6 +2429,27 @@ const (
NodeSelectorOpLt NodeSelectorOperator = "Lt"
)
// A topology selector term represents the result of label queries.
// A null or empty topology selector term matches no objects.
// The requirements of them are ANDed.
// It provides a subset of functionality as NodeSelectorTerm.
// This is an alpha feature and may change in the future.
type TopologySelectorTerm struct {
// A list of topology selector requirements by labels.
// +optional
MatchLabelExpressions []TopologySelectorLabelRequirement `json:"matchLabelExpressions,omitempty" protobuf:"bytes,1,rep,name=matchLabelExpressions"`
}
// A topology selector requirement is a selector that matches given label.
// This is an alpha feature and may change in the future.
type TopologySelectorLabelRequirement struct {
// The label key that the selector applies to.
Key string `json:"key" protobuf:"bytes,1,opt,name=key"`
// An array of string values. One value must match the label to be selected.
// Each entry in Values is ORed.
Values []string `json:"values" protobuf:"bytes,2,rep,name=values"`
}
// Affinity is a group of affinity scheduling rules.
type Affinity struct {
// Describes node affinity scheduling rules for the pod.

View File

@@ -66,6 +66,14 @@ type StorageClass struct {
// the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

View File

@@ -66,6 +66,14 @@ type StorageClass struct {
// the VolumeScheduling feature.
// +optional
VolumeBindingMode *VolumeBindingMode `json:"volumeBindingMode,omitempty" protobuf:"bytes,7,opt,name=volumeBindingMode"`
// Restrict the node topologies where volumes can be dynamically provisioned.
// Each volume plugin defines its own supported topology specifications.
// An empty TopologySelectorTerm list means there is no topology restriction.
// This field is alpha-level and is only honored by servers that enable
// the DynamicProvisioningScheduling feature.
// +optional
AllowedTopologies []v1.TopologySelectorTerm `json:"allowedTopologies,omitempty" protobuf:"bytes,8,rep,name=allowedTopologies"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object