diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index d12fc963747..b6890e52166 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -66,6 +66,7 @@ go_test( "//pkg/scheduler/framework/plugins:go_default_library", "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", + "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library", "//pkg/scheduler/framework/plugins/nodelabel:go_default_library", "//pkg/scheduler/framework/plugins/nodeports:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index ef456785e03..7da134bb625 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -1578,6 +1578,48 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { "PreBindPlugin": {{Name: "VolumeBinding"}}, "BindPlugin": {{Name: "DefaultBinder"}}, } + defaultPluginConfigs := []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{ + MinCandidateNodesPercentage: 10, + MinCandidateNodesAbsolute: 100, + }, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{}, + }, + { + Name: "NodeResourcesFit", + Args: &config.NodeResourcesFitArgs{}, + }, + { + Name: "NodeResourcesLeastAllocated", + Args: &config.NodeResourcesLeastAllocatedArgs{ + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, + { + Name: "PodTopologySpread", + Args: &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting}, + }, + { + Name: "VolumeBinding", + Args: &config.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, + } testcases := []struct { name string @@ -1589,11 +1631,76 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { { name: "default plugins", wantPlugins: defaultPlugins, - wantPluginConfig: nil, + wantPluginConfig: defaultPluginConfigs, }, { - name: "default plugins with customized plugin config", - wantPlugins: defaultPlugins, + name: "in-tree plugins with customized plugin config", + plugins: config.Plugins{ + Filter: &config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "NodeLabel"}, + {Name: "ServiceAffinity"}, + }, + }, + Score: &config.PluginSet{ + Enabled: []config.Plugin{ + {Name: "RequestedToCapacityRatio"}, + }, + }, + }, + wantPlugins: map[string][]config.Plugin{ + "QueueSortPlugin": { + {Name: "PrioritySort"}, + }, + "PreFilterPlugin": { + {Name: "NodeResourcesFit"}, + {Name: "NodePorts"}, + {Name: "PodTopologySpread"}, + {Name: "InterPodAffinity"}, + {Name: "VolumeBinding"}, + }, + "FilterPlugin": { + {Name: "NodeUnschedulable"}, + {Name: "NodeName"}, + {Name: "TaintToleration"}, + {Name: "NodeAffinity"}, + {Name: "NodePorts"}, + {Name: "NodeResourcesFit"}, + {Name: "VolumeRestrictions"}, + {Name: "EBSLimits"}, + {Name: "GCEPDLimits"}, + {Name: "NodeVolumeLimits"}, + {Name: "AzureDiskLimits"}, + {Name: "VolumeBinding"}, + {Name: "VolumeZone"}, + {Name: "PodTopologySpread"}, + {Name: "InterPodAffinity"}, + {Name: "NodeLabel"}, + {Name: "ServiceAffinity"}, + }, + "PostFilterPlugin": { + {Name: "DefaultPreemption"}, + }, + "PreScorePlugin": { + {Name: "InterPodAffinity"}, + {Name: "PodTopologySpread"}, + {Name: "TaintToleration"}, + }, + "ScorePlugin": { + {Name: "NodeResourcesBalancedAllocation", Weight: 1}, + {Name: "ImageLocality", Weight: 1}, + {Name: "InterPodAffinity", Weight: 1}, + {Name: "NodeResourcesLeastAllocated", Weight: 1}, + {Name: "NodeAffinity", Weight: 1}, + {Name: "NodePreferAvoidPods", Weight: 10000}, + {Name: "PodTopologySpread", Weight: 2}, + {Name: "TaintToleration", Weight: 1}, + {Name: "RequestedToCapacityRatio", Weight: 1}, + }, + "ReservePlugin": {{Name: "VolumeBinding"}}, + "PreBindPlugin": {{Name: "VolumeBinding"}}, + "BindPlugin": {{Name: "DefaultBinder"}}, + }, pluginConfig: []config.PluginConfig{ { Name: "NodeResourcesFit", @@ -1660,12 +1767,47 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { }, }, wantPluginConfig: []config.PluginConfig{ + { + Name: "DefaultPreemption", + Args: &config.DefaultPreemptionArgs{ + MinCandidateNodesPercentage: 10, + MinCandidateNodesAbsolute: 100, + }, + }, + { + Name: "InterPodAffinity", + Args: &config.InterPodAffinityArgs{ + HardPodAffinityWeight: 100, + }, + }, + { + Name: "NodeAffinity", + Args: &config.NodeAffinityArgs{}, + }, + { + Name: "NodeLabel", + Args: &config.NodeLabelArgs{ + PresentLabels: []string{"foo", "bar"}, + AbsentLabels: []string{"apple"}, + PresentLabelsPreference: []string{"dog"}, + AbsentLabelsPreference: []string{"cat"}, + }, + }, { Name: "NodeResourcesFit", Args: &config.NodeResourcesFitArgs{ IgnoredResources: []string{"foo", "bar"}, }, }, + { + Name: "NodeResourcesLeastAllocated", + Args: &config.NodeResourcesLeastAllocatedArgs{ + Resources: []config.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, { Name: "PodTopologySpread", Args: &config.PodTopologySpreadArgs{ @@ -1695,21 +1837,6 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { }, }, }, - { - Name: "InterPodAffinity", - Args: &config.InterPodAffinityArgs{ - HardPodAffinityWeight: 100, - }, - }, - { - Name: "NodeLabel", - Args: &config.NodeLabelArgs{ - PresentLabels: []string{"foo", "bar"}, - AbsentLabels: []string{"apple"}, - PresentLabelsPreference: []string{"dog"}, - AbsentLabelsPreference: []string{"cat"}, - }, - }, { Name: "ServiceAffinity", Args: &config.ServiceAffinityArgs{ @@ -1849,6 +1976,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { }, PreScore: &config.PluginSet{ Enabled: []config.Plugin{ + {Name: "PodTopologySpread"}, {Name: "TaintToleration"}, {Name: "SelectorSpread"}, {Name: "InterPodAffinity"}, @@ -1859,6 +1987,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { }, Score: &config.PluginSet{ Enabled: []config.Plugin{ + {Name: "PodTopologySpread", Weight: 24}, {Name: "TaintToleration", Weight: 24}, {Name: "SelectorSpread", Weight: 24}, {Name: "NodePreferAvoidPods", Weight: 24}, @@ -1906,11 +2035,13 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { {Name: "DefaultPreemption"}, }, "PreScorePlugin": { + {Name: "PodTopologySpread"}, {Name: "TaintToleration"}, {Name: "SelectorSpread"}, {Name: "InterPodAffinity"}, }, "ScorePlugin": { + {Name: "PodTopologySpread", Weight: 24}, {Name: "TaintToleration", Weight: 24}, {Name: "SelectorSpread", Weight: 24}, {Name: "NodePreferAvoidPods", Weight: 24}, @@ -1924,7 +2055,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) { "PreBindPlugin": {{Name: "VolumeBinding"}}, "BindPlugin": {{Name: "DefaultBinder"}}, }, - wantPluginConfig: nil, + wantPluginConfig: defaultPluginConfigs, }, } for _, tc := range testcases { diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 90e52bf5a94..01131958634 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -137,12 +137,8 @@ func (c *Configurator) create() (*Scheduler, error) { frameworkruntime.WithSnapshotSharedLister(c.nodeInfoSnapshot), frameworkruntime.WithRunAllFilters(c.alwaysCheckAllPredicates), frameworkruntime.WithPodNominator(nominator), + frameworkruntime.WithCaptureProfile(frameworkruntime.CaptureProfile(c.frameworkCapturer)), ) - for _, p := range c.profiles { - if c.frameworkCapturer != nil { - c.frameworkCapturer(p) - } - } if err != nil { return nil, fmt.Errorf("initializing profiles: %v", err) } diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index c8debd81e91..551c19c5948 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -39,10 +39,12 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework" frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodelabel" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" @@ -105,10 +107,39 @@ func TestCreateFromConfig(t *testing.T) { "apiVersion" : "v1" }`), wantPluginConfig: []schedulerapi.PluginConfig{ + { + Name: interpodaffinity.Name, + Args: &schedulerapi.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: nodeaffinity.Name, + Args: &schedulerapi.NodeAffinityArgs{}, + }, + { + Name: noderesources.FitName, + Args: &schedulerapi.NodeResourcesFitArgs{}, + }, + { + Name: noderesources.LeastAllocatedName, + Args: &schedulerapi.NodeResourcesLeastAllocatedArgs{ + Resources: []schedulerapi.ResourceSpec{ + {Name: "cpu", Weight: 1}, + {Name: "memory", Weight: 1}, + }, + }, + }, { Name: podtopologyspread.Name, Args: &schedulerapi.PodTopologySpreadArgs{DefaultingType: schedulerapi.SystemDefaulting}, }, + { + Name: volumebinding.Name, + Args: &schedulerapi.VolumeBindingArgs{ + BindTimeoutSeconds: 600, + }, + }, }, wantPlugins: &schedulerapi.Plugins{ QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}}, @@ -200,6 +231,16 @@ func TestCreateFromConfig(t *testing.T) { ] }`), wantPluginConfig: []schedulerapi.PluginConfig{ + { + Name: interpodaffinity.Name, + Args: &schedulerapi.InterPodAffinityArgs{ + HardPodAffinityWeight: 1, + }, + }, + { + Name: nodeaffinity.Name, + Args: &schedulerapi.NodeAffinityArgs{}, + }, { Name: nodelabel.Name, Args: &schedulerapi.NodeLabelArgs{ @@ -209,13 +250,6 @@ func TestCreateFromConfig(t *testing.T) { AbsentLabelsPreference: []string{"l2"}, }, }, - { - Name: serviceaffinity.Name, - Args: &schedulerapi.ServiceAffinityArgs{ - AffinityLabels: []string{"zone", "foo"}, - AntiAffinityLabelsPreference: []string{"rack", "zone"}, - }, - }, { Name: noderesources.RequestedToCapacityRatioName, Args: &schedulerapi.RequestedToCapacityRatioArgs{ @@ -226,6 +260,13 @@ func TestCreateFromConfig(t *testing.T) { Resources: []schedulerapi.ResourceSpec{}, }, }, + { + Name: serviceaffinity.Name, + Args: &schedulerapi.ServiceAffinityArgs{ + AffinityLabels: []string{"zone", "foo"}, + AntiAffinityLabelsPreference: []string{"rack", "zone"}, + }, + }, }, wantPlugins: &schedulerapi.Plugins{ QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}}, @@ -262,46 +303,31 @@ func TestCreateFromConfig(t *testing.T) { "kind" : "Policy", "apiVersion" : "v1", "predicates" : [ - {"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}}, - {"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}}, {"name" : "PodFitsResources"}, {"name" : "PodFitsHostPorts"} ], "priorities" : [ - {"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}}, - {"name" : "NodeAffinityPriority", "weight" : 2}, - {"name" : "ImageLocalityPriority", "weight" : 1}, {"name" : "InterPodAffinityPriority", "weight" : 1} ], "hardPodAffinitySymmetricWeight" : 10 }`), wantPluginConfig: []schedulerapi.PluginConfig{ - { - Name: nodelabel.Name, - Args: &schedulerapi.NodeLabelArgs{ - PresentLabels: []string{"zone"}, - }, - }, - { - Name: serviceaffinity.Name, - Args: &schedulerapi.ServiceAffinityArgs{ - AffinityLabels: []string{"zone"}, - AntiAffinityLabelsPreference: []string{"rack"}, - }, - }, { Name: interpodaffinity.Name, Args: &schedulerapi.InterPodAffinityArgs{ HardPodAffinityWeight: 10, }, }, + { + Name: "NodeResourcesFit", + Args: &schedulerapi.NodeResourcesFitArgs{}, + }, }, wantPlugins: &schedulerapi.Plugins{ QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}}, PreFilter: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{ {Name: "NodePorts"}, {Name: "NodeResourcesFit"}, - {Name: "ServiceAffinity"}, }}, Filter: &schedulerapi.PluginSet{ Enabled: []schedulerapi.Plugin{ @@ -309,18 +335,13 @@ func TestCreateFromConfig(t *testing.T) { {Name: "NodePorts"}, {Name: "NodeResourcesFit"}, {Name: "TaintToleration"}, - {Name: "NodeLabel"}, - {Name: "ServiceAffinity"}, }, }, PostFilter: &schedulerapi.PluginSet{}, PreScore: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "InterPodAffinity"}}}, Score: &schedulerapi.PluginSet{ Enabled: []schedulerapi.Plugin{ - {Name: "ImageLocality", Weight: 1}, {Name: "InterPodAffinity", Weight: 1}, - {Name: "NodeAffinity", Weight: 2}, - {Name: "ServiceAffinity", Weight: 3}, }, }, Reserve: &schedulerapi.PluginSet{}, diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 18d3117634f..87e0afd2b13 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "sort" "time" v1 "k8s.io/api/core/v1" @@ -130,6 +131,7 @@ type frameworkOptions struct { podNominator framework.PodNominator extenders []framework.Extender runAllFilters bool + captureProfile CaptureProfile } // Option for the frameworkImpl. @@ -199,6 +201,16 @@ func WithExtenders(extenders []framework.Extender) Option { } } +// CaptureProfile is a callback to capture a finalized profile. +type CaptureProfile func(config.KubeSchedulerProfile) + +// WithCaptureProfile sets a callback to capture the finalized profile. +func WithCaptureProfile(c CaptureProfile) Option { + return func(o *frameworkOptions) { + o.captureProfile = c + } +} + var defaultFrameworkOptions = frameworkOptions{ metricsRecorder: newMetricsRecorder(1000, time.Second), } @@ -258,6 +270,11 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi } pluginConfig[name] = args[i].Args } + outputProfile := config.KubeSchedulerProfile{ + SchedulerName: f.profileName, + Plugins: plugins, + PluginConfig: make([]config.PluginConfig, 0, len(pg)), + } pluginsMap := make(map[string]framework.Plugin) var totalPriority int64 @@ -271,6 +288,12 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi if err != nil { return nil, fmt.Errorf("getting args for Plugin %q: %w", name, err) } + if args != nil { + outputProfile.PluginConfig = append(outputProfile.PluginConfig, config.PluginConfig{ + Name: name, + Args: args, + }) + } p, err := factory(args, f) if err != nil { return nil, fmt.Errorf("initializing plugin %q: %w", name, err) @@ -314,6 +337,17 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi return nil, fmt.Errorf("at least one bind plugin is needed") } + if options.captureProfile != nil { + if len(outputProfile.PluginConfig) != 0 { + sort.Slice(outputProfile.PluginConfig, func(i, j int) bool { + return outputProfile.PluginConfig[i].Name < outputProfile.PluginConfig[j].Name + }) + } else { + outputProfile.PluginConfig = nil + } + options.captureProfile(outputProfile) + } + return f, nil }