github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move scheduler plugin set and configuration defaulting to component config

Browse Source
This commit is contained in:
Abdullah Gharaibeh
2021-06-10 08:45:49 -04:00
parent 21ee533508
commit 265ef1741f
37 changed files with 3200 additions and 3382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
test/integration/scheduler/scheduler_test.go
View File

@@ -28,6 +28,7 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
@@ -36,12 +37,15 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/events"
"k8s.io/kube-scheduler/config/v1beta2"
"k8s.io/kubernetes/pkg/scheduler"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/profile"
st "k8s.io/kubernetes/pkg/scheduler/testing"
"k8s.io/kubernetes/test/integration/framework"
testutils "k8s.io/kubernetes/test/integration/util"
"k8s.io/utils/pointer"
)
type nodeMutationFunc func(t *testing.T, n *v1.Node, nodeLister corelisters.NodeLister, c clientset.Interface)
@@ -67,7 +71,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
for i, test := range []struct {
policy string
expectedPlugins map[string][]kubeschedulerconfig.Plugin
expectedPlugins config.Plugins
}{
{
policy: `{
@@ -80,21 +84,17 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{"name" : "ImageLocalityPriority", "weight" : 1}
]
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
{Name: "NodeResourcesFit"},
},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "NodeResourcesFit"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"ScorePlugin": {
{Name: "ImageLocality", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Score: config.PluginSet{Enabled: []config.Plugin{{Name: "ImageLocality", Weight: 1}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@@ -102,17 +102,17 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"kind" : "Policy",
"apiVersion" : "v1"
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
{Name: "VolumeBinding"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "NodeName"},
@@ -128,15 +128,15 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeAffinity"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 2},
{Name: "ImageLocality", Weight: 1},
@@ -145,10 +145,10 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "TaintToleration", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@@ -158,14 +158,14 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
"predicates" : [],
"priorities" : []
}`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@@ -177,38 +177,38 @@ priorities:
- name: ImageLocalityPriority
weight: 1
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"ScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "ImageLocality", Weight: 1},
},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
policy: `apiVersion: v1
kind: Policy
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"PreFilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
PreFilter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesFit"},
{Name: "NodePorts"},
{Name: "NodeAffinity"},
{Name: "VolumeBinding"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"FilterPlugin": {
}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "NodeResourcesFit"},
{Name: "NodeName"},
@@ -224,15 +224,15 @@ kind: Policy
{Name: "VolumeZone"},
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"PreScorePlugin": {
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
PreScore: config.PluginSet{Enabled: []config.Plugin{
{Name: "PodTopologySpread"},
{Name: "InterPodAffinity"},
{Name: "NodeAffinity"},
{Name: "TaintToleration"},
},
"ScorePlugin": {
}},
Score: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeResourcesBalancedAllocation", Weight: 1},
{Name: "PodTopologySpread", Weight: 2},
{Name: "ImageLocality", Weight: 1},
@@ -241,10 +241,10 @@ kind: Policy
{Name: "NodeAffinity", Weight: 1},
{Name: "NodePreferAvoidPods", Weight: 10000},
{Name: "TaintToleration", Weight: 1},
},
"ReservePlugin": {{Name: "VolumeBinding"}},
"PreBindPlugin": {{Name: "VolumeBinding"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
Reserve: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
PreBind: config.PluginSet{Enabled: []config.Plugin{{Name: "VolumeBinding"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
{
@@ -253,14 +253,14 @@ kind: Policy
predicates: []
priorities: []
`,
expectedPlugins: map[string][]kubeschedulerconfig.Plugin{
"QueueSortPlugin": {{Name: "PrioritySort"}},
"FilterPlugin": {
expectedPlugins: config.Plugins{
QueueSort: config.PluginSet{Enabled: []config.Plugin{{Name: "PrioritySort"}}},
Filter: config.PluginSet{Enabled: []config.Plugin{
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},
},
"PostFilterPlugin": {{Name: "DefaultPreemption"}},
"BindPlugin": {{Name: "DefaultBinder"}},
}},
PostFilter: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultPreemption"}}},
Bind: config.PluginSet{Enabled: []config.Plugin{{Name: "DefaultBinder"}}},
},
},
} {
@@ -268,7 +268,7 @@ priorities: []
configPolicyName := fmt.Sprintf("scheduler-custom-policy-config-%d", i)
policyConfigMap := v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: configPolicyName},
Data: map[string]string{kubeschedulerconfig.SchedulerPolicyConfigMapKey: test.policy},
Data: map[string]string{config.SchedulerPolicyConfigMapKey: test.policy},
}
policyConfigMap.APIVersion = "v1"
@@ -282,30 +282,20 @@ priorities: []
informerFactory,
profile.NewRecorderFactory(eventBroadcaster),
nil,
scheduler.WithLegacyPolicySource(&kubeschedulerconfig.SchedulerPolicySource{
ConfigMap: &kubeschedulerconfig.SchedulerPolicyConfigMapSource{
scheduler.WithProfiles([]config.KubeSchedulerProfile(nil)...),
scheduler.WithLegacyPolicySource(&config.SchedulerPolicySource{
ConfigMap: &config.SchedulerPolicyConfigMapSource{
Namespace: policyConfigMap.Namespace,
Name: policyConfigMap.Name,
},
}),
scheduler.WithProfiles(kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 30,
},
},
},
}),
)
if err != nil {
t.Fatalf("couldn't make scheduler config for test %d: %v", i, err)
}
schedPlugins := sched.Profiles[v1.DefaultSchedulerName].ListPlugins()
if diff := cmp.Diff(test.expectedPlugins, schedPlugins); diff != "" {
if diff := cmp.Diff(&test.expectedPlugins, schedPlugins); diff != "" {
t.Errorf("unexpected plugins diff (-want, +got): %s", diff)
}
}
@@ -329,27 +319,33 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) {
stopCh := make(chan struct{})
eventBroadcaster.StartRecordingToSink(stopCh)
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(v1.DefaultSchedulerName),
PluginConfig: []v1beta2.PluginConfig{
{
Name: "VolumeBinding",
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(30),
},
},
},
}},
},
})
_, err := scheduler.New(clientSet,
informerFactory,
profile.NewRecorderFactory(eventBroadcaster),
nil,
scheduler.WithLegacyPolicySource(&kubeschedulerconfig.SchedulerPolicySource{
ConfigMap: &kubeschedulerconfig.SchedulerPolicyConfigMapSource{
scheduler.WithLegacyPolicySource(&config.SchedulerPolicySource{
ConfigMap: &config.SchedulerPolicyConfigMapSource{
Namespace: "non-existent-config",
Name: "non-existent-config",
},
}),
scheduler.WithProfiles(kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: v1.DefaultSchedulerName,
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 30,
},
},
},
}),
scheduler.WithProfiles(cfg.Profiles...),
)
if err == nil {
@@ -560,8 +556,22 @@ func TestMultipleSchedulers(t *testing.T) {
}
// 5. create and start a scheduler with name "foo-scheduler"
fooProf := kubeschedulerconfig.KubeSchedulerProfile{SchedulerName: fooScheduler}
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, nil, scheduler.WithProfiles(fooProf))
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{{
SchedulerName: pointer.StringPtr(fooScheduler),
PluginConfig: []v1beta2.PluginConfig{
{
Name: "VolumeBinding",
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(30),
},
},
},
}},
},
})
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, nil, scheduler.WithProfiles(cfg.Profiles...))
testutils.SyncInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.Ctx)
@@ -622,14 +632,14 @@ func TestMultipleSchedulers(t *testing.T) {
}
func TestMultipleSchedulingProfiles(t *testing.T) {
testCtx := initTest(t, "multi-scheduler", scheduler.WithProfiles(
kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: "default-scheduler",
cfg := configtesting.V1beta2ToInternalWithDefaults(t, v1beta2.KubeSchedulerConfiguration{
Profiles: []v1beta2.KubeSchedulerProfile{
{SchedulerName: pointer.StringPtr("default-scheduler")},
{SchedulerName: pointer.StringPtr("custom-scheduler")},
},
kubeschedulerconfig.KubeSchedulerProfile{
SchedulerName: "custom-scheduler",
},
))
})
testCtx := initTest(t, "multi-scheduler", scheduler.WithProfiles(cfg.Profiles...))
defer testutils.CleanupTest(t, testCtx)
node := &v1.Node{