Merge pull request #95191 from alculquicondor/policy-default-spread

Move predicates and priorities configuration creation to Policy mapping

pkg/scheduler/factory.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"sort"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
@@ -268,138 +267,56 @@ func (c *Configurator) createFromConfig(policy schedulerapi.Policy) (*Scheduler,
 
 	klog.V(2).Infof("Creating scheduler with fit predicates '%v' and priority functions '%v'", predicateKeys, priorityKeys)
 
-	pluginsForPredicates, pluginConfigForPredicates, err := getPredicateConfigs(predicateKeys, lr, args)
-	if err != nil {
-		return nil, err
-	}
-
-	pluginsForPriorities, pluginConfigForPriorities, err := getPriorityConfigs(priorityKeys, lr, args)
-	if err != nil {
-		return nil, err
-	}
 	// Combine all framework configurations. If this results in any duplication, framework
 	// instantiation should fail.
-	var defPlugins schedulerapi.Plugins
+
 	// "PrioritySort" and "DefaultBinder" were neither predicates nor priorities
 	// before. We add them by default.
-	defPlugins.Append(&schedulerapi.Plugins{
+	plugins := schedulerapi.Plugins{
 		QueueSort: &schedulerapi.PluginSet{
 			Enabled: []schedulerapi.Plugin{{Name: queuesort.Name}},
 		},
 		Bind: &schedulerapi.PluginSet{
 			Enabled: []schedulerapi.Plugin{{Name: defaultbinder.Name}},
 		},
-	})
+	}
-	defPlugins.Append(pluginsForPredicates)
+	var pluginConfig []schedulerapi.PluginConfig
-	defPlugins.Append(pluginsForPriorities)
+	var err error
-	defPluginConfig, err := mergePluginConfigsFromPolicy(pluginConfigForPredicates, pluginConfigForPriorities)
+	if plugins, pluginConfig, err = lr.AppendPredicateConfigs(predicateKeys, args, plugins, pluginConfig); err != nil {
-	if err != nil {
+		return nil, err
+	}
+	if plugins, pluginConfig, err = lr.AppendPriorityConfigs(priorityKeys, args, plugins, pluginConfig); err != nil {
+		return nil, err
+	}
+	if pluginConfig, err = dedupPluginConfigs(pluginConfig); err != nil {
 		return nil, err
 	}
 	for i := range c.profiles {
 		prof := &c.profiles[i]
-		// Plugins are empty when using Policy.
+		// Plugins and PluginConfig are empty when using Policy; overriding.
 		prof.Plugins = &schedulerapi.Plugins{}
-		prof.Plugins.Append(&defPlugins)
+		prof.Plugins.Append(&plugins)
-
+		prof.PluginConfig = pluginConfig
-		// PluginConfig is ignored when using Policy.
-		prof.PluginConfig = defPluginConfig
 	}
 
 	return c.create()
 }
 
-// mergePluginConfigsFromPolicy merges the giving plugin configs ensuring that,
+// dedupPluginConfigs removes duplicates from pluginConfig, ensuring that,
 // if a plugin name is repeated, the arguments are the same.
-func mergePluginConfigsFromPolicy(pc1, pc2 []schedulerapi.PluginConfig) ([]schedulerapi.PluginConfig, error) {
+func dedupPluginConfigs(pc []schedulerapi.PluginConfig) ([]schedulerapi.PluginConfig, error) {
 	args := make(map[string]runtime.Object)
-	for _, c := range pc1 {
+	result := make([]schedulerapi.PluginConfig, 0, len(pc))
-		args[c.Name] = c.Args
+	for _, c := range pc {
-	}
+		if v, found := args[c.Name]; !found {
-	for _, c := range pc2 {
+			result = append(result, c)
-		if v, ok := args[c.Name]; ok && !cmp.Equal(v, c.Args) {
+			args[c.Name] = c.Args
+		} else if !cmp.Equal(v, c.Args) {
 			// This should be unreachable.
 			return nil, fmt.Errorf("inconsistent configuration produced for plugin %s", c.Name)
 		}
-		args[c.Name] = c.Args
 	}
-	pc := make([]schedulerapi.PluginConfig, 0, len(args))
+	return result, nil
-	for k, v := range args {
-		pc = append(pc, schedulerapi.PluginConfig{
-			Name: k,
-			Args: v,
-		})
-	}
-	return pc, nil
-}
-
-// getPriorityConfigs returns priorities configuration: ones that will run as priorities and ones that will run
-// as framework plugins. Specifically, a priority will run as a framework plugin if a plugin config producer was
-// registered for that priority.
-func getPriorityConfigs(keys map[string]int64, lr *frameworkplugins.LegacyRegistry, args *frameworkplugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) {
-	var plugins schedulerapi.Plugins
-	var pluginConfig []schedulerapi.PluginConfig
-
-	// Sort the keys so that it is easier for unit tests to do compare.
-	var sortedKeys []string
-	for k := range keys {
-		sortedKeys = append(sortedKeys, k)
-	}
-	sort.Strings(sortedKeys)
-
-	for _, priority := range sortedKeys {
-		weight := keys[priority]
-		producer, exist := lr.PriorityToConfigProducer[priority]
-		if !exist {
-			return nil, nil, fmt.Errorf("no config producer registered for %q", priority)
-		}
-		a := *args
-		a.Weight = int32(weight)
-		pl, plc := producer(a)
-		plugins.Append(&pl)
-		pluginConfig = append(pluginConfig, plc...)
-	}
-	return &plugins, pluginConfig, nil
-}
-
-// getPredicateConfigs returns predicates configuration: ones that will run as fitPredicates and ones that will run
-// as framework plugins. Specifically, a predicate will run as a framework plugin if a plugin config producer was
-// registered for that predicate.
-// Note that the framework executes plugins according to their order in the Plugins list, and so predicates run as plugins
-// are added to the Plugins list according to the order specified in predicates.Ordering().
-func getPredicateConfigs(keys sets.String, lr *frameworkplugins.LegacyRegistry, args *frameworkplugins.ConfigProducerArgs) (*schedulerapi.Plugins, []schedulerapi.PluginConfig, error) {
-	allPredicates := keys.Union(lr.MandatoryPredicates)
-
-	// Create the framework plugin configurations, and place them in the order
-	// that the corresponding predicates were supposed to run.
-	var plugins schedulerapi.Plugins
-	var pluginConfig []schedulerapi.PluginConfig
-
-	for _, predicateKey := range frameworkplugins.PredicateOrdering() {
-		if allPredicates.Has(predicateKey) {
-			producer, exist := lr.PredicateToConfigProducer[predicateKey]
-			if !exist {
-				return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey)
-			}
-			pl, plc := producer(*args)
-			plugins.Append(&pl)
-			pluginConfig = append(pluginConfig, plc...)
-			allPredicates.Delete(predicateKey)
-		}
-	}
-
-	// Third, add the rest in no specific order.
-	for predicateKey := range allPredicates {
-		producer, exist := lr.PredicateToConfigProducer[predicateKey]
-		if !exist {
-			return nil, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey)
-		}
-		pl, plc := producer(*args)
-		plugins.Append(&pl)
-		pluginConfig = append(pluginConfig, plc...)
-	}
-
-	return &plugins, pluginConfig, nil
 }
 
 // MakeDefaultErrorFunc construct a function to handle pod scheduler error

pkg/scheduler/framework/plugins/BUILD

@@ -80,6 +80,9 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/scheduler/apis/config:go_default_library",
+        "//pkg/scheduler/framework/plugins/nodeunschedulable:go_default_library",
+        "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/github.com/google/go-cmp/cmp:go_default_library",
     ],
 )

pkg/scheduler/framework/plugins/legacy_registry.go

@@ -17,8 +17,10 @@ limitations under the License.
 package plugins
 
 import (
-	"k8s.io/apimachinery/pkg/util/sets"
+	"fmt"
+	"sort"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
@@ -127,24 +129,24 @@ const (
 	EvenPodsSpreadPred = "EvenPodsSpread"
 )
 
-// PredicateOrdering returns the ordering of predicate execution.
+// predicateOrdering is the ordering of predicate execution.
-func PredicateOrdering() []string {
+var predicateOrdering = []string{
-	return []string{CheckNodeUnschedulablePred,
+	CheckNodeUnschedulablePred,
-		GeneralPred, HostNamePred, PodFitsHostPortsPred,
+	GeneralPred, HostNamePred, PodFitsHostPortsPred,
-		MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred,
+	MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred,
-		PodToleratesNodeTaintsPred, CheckNodeLabelPresencePred,
+	PodToleratesNodeTaintsPred, CheckNodeLabelPresencePred,
-		CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred,
+	CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred,
-		MaxAzureDiskVolumeCountPred, MaxCinderVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred,
+	MaxAzureDiskVolumeCountPred, MaxCinderVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred,
-		EvenPodsSpreadPred, MatchInterPodAffinityPred}
+	EvenPodsSpreadPred, MatchInterPodAffinityPred,
 }
 
 // LegacyRegistry is used to store current state of registered predicates and priorities.
 type LegacyRegistry struct {
 	// maps that associate predicates/priorities with framework plugin configurations.
-	PredicateToConfigProducer map[string]ConfigProducer
+	predicateToConfigProducer map[string]configProducer
-	PriorityToConfigProducer  map[string]ConfigProducer
+	priorityToConfigProducer  map[string]configProducer
 	// predicates that will always be configured.
-	MandatoryPredicates sets.String
+	mandatoryPredicates sets.String
 	// predicates and priorities that will be used if either was set to nil in a
 	// given v1.Policy configuration.
 	DefaultPredicates sets.String
@@ -169,16 +171,16 @@ type ConfigProducerArgs struct {
 	InterPodAffinityArgs *config.InterPodAffinityArgs
 }
 
-// ConfigProducer returns the set of plugins and their configuration for a
+// configProducer appends the set of plugins and their configuration for a
 // predicate/priority given the args.
-type ConfigProducer func(args ConfigProducerArgs) (config.Plugins, []config.PluginConfig)
+type configProducer func(ConfigProducerArgs, *config.Plugins, *[]config.PluginConfig)
 
 // NewLegacyRegistry returns a legacy algorithm registry of predicates and priorities.
 func NewLegacyRegistry() *LegacyRegistry {
 	registry := &LegacyRegistry{
-		// MandatoryPredicates the set of keys for predicates that the scheduler will
+		// mandatoryPredicates the set of keys for predicates that the scheduler will
 		// be configured with all the time.
-		MandatoryPredicates: sets.NewString(
+		mandatoryPredicates: sets.NewString(
 			PodToleratesNodeTaintsPred,
 			CheckNodeUnschedulablePred,
 		),
@@ -212,248 +214,275 @@ func NewLegacyRegistry() *LegacyRegistry {
 			EvenPodsSpreadPriority:      2,
 		},
 
-		PredicateToConfigProducer: make(map[string]ConfigProducer),
+		predicateToConfigProducer: make(map[string]configProducer),
-		PriorityToConfigProducer:  make(map[string]ConfigProducer),
+		priorityToConfigProducer:  make(map[string]configProducer),
 	}
 
 	registry.registerPredicateConfigProducer(GeneralPred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			// GeneralPredicate is a combination of predicates.
 			plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil)
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, noderesources.FitName, nil)
 			if args.NodeResourcesFitArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: noderesources.FitName, Args: args.NodeResourcesFitArgs})
 			}
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil)
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil)
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil)
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(PodToleratesNodeTaintsPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, tainttoleration.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(PodFitsResourcesPred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil)
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, noderesources.FitName, nil)
 			if args.NodeResourcesFitArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: noderesources.FitName, Args: args.NodeResourcesFitArgs})
 			}
-			return
 		})
 	registry.registerPredicateConfigProducer(HostNamePred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(PodFitsHostPortsPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil)
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, nodeports.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MatchNodeSelectorPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(CheckNodeUnschedulablePred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodeunschedulable.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(CheckVolumeBindingPred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, volumebinding.Name, nil)
 			plugins.Filter = appendToPluginSet(plugins.Filter, volumebinding.Name, nil)
 			plugins.Reserve = appendToPluginSet(plugins.Reserve, volumebinding.Name, nil)
 			plugins.PreBind = appendToPluginSet(plugins.PreBind, volumebinding.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(NoDiskConflictPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, volumerestrictions.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(NoVolumeZoneConflictPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, volumezone.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MaxCSIVolumeCountPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CSIName, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MaxEBSVolumeCountPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.EBSName, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MaxGCEPDVolumeCountPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.GCEPDName, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MaxAzureDiskVolumeCountPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.AzureDiskName, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MaxCinderVolumeCountPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodevolumelimits.CinderName, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(MatchInterPodAffinityPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, interpodaffinity.Name, nil)
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, interpodaffinity.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(CheckNodeLabelPresencePred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, nodelabel.Name, nil)
 			if args.NodeLabelArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: nodelabel.Name, Args: args.NodeLabelArgs})
 			}
-			return
 		})
 	registry.registerPredicateConfigProducer(CheckServiceAffinityPred,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, serviceaffinity.Name, nil)
 			if args.ServiceAffinityArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: serviceaffinity.Name, Args: args.ServiceAffinityArgs})
 			}
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, serviceaffinity.Name, nil)
-			return
 		})
 	registry.registerPredicateConfigProducer(EvenPodsSpreadPred,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.PreFilter = appendToPluginSet(plugins.PreFilter, podtopologyspread.Name, nil)
 			plugins.Filter = appendToPluginSet(plugins.Filter, podtopologyspread.Name, nil)
-			return
 		})
 
 	// Register Priorities.
 	registry.registerPriorityConfigProducer(SelectorSpreadPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, selectorspread.Name, &args.Weight)
 			plugins.PreScore = appendToPluginSet(plugins.PreScore, selectorspread.Name, nil)
-			return
 		})
 	registry.registerPriorityConfigProducer(TaintTolerationPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.PreScore = appendToPluginSet(plugins.PreScore, tainttoleration.Name, nil)
 			plugins.Score = appendToPluginSet(plugins.Score, tainttoleration.Name, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(NodeAffinityPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, nodeaffinity.Name, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(ImageLocalityPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(InterPodAffinityPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			plugins.PreScore = appendToPluginSet(plugins.PreScore, interpodaffinity.Name, nil)
 			plugins.Score = appendToPluginSet(plugins.Score, interpodaffinity.Name, &args.Weight)
 			if args.InterPodAffinityArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: interpodaffinity.Name, Args: args.InterPodAffinityArgs})
 			}
-			return
 		})
 	registry.registerPriorityConfigProducer(NodePreferAvoidPodsPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(MostRequestedPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, noderesources.MostAllocatedName, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(BalancedResourceAllocation,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, noderesources.BalancedAllocationName, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(LeastRequestedPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, noderesources.LeastAllocatedName, &args.Weight)
-			return
 		})
 	registry.registerPriorityConfigProducer(noderesources.RequestedToCapacityRatioName,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, noderesources.RequestedToCapacityRatioName, &args.Weight)
 			if args.RequestedToCapacityRatioArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: noderesources.RequestedToCapacityRatioName, Args: args.RequestedToCapacityRatioArgs})
 			}
-			return
 		})
 	registry.registerPriorityConfigProducer(nodelabel.Name,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			// If there are n LabelPreference priorities in the policy, the weight for the corresponding
 			// score plugin is n*weight (note that the validation logic verifies that all LabelPreference
 			// priorities specified in Policy have the same weight).
 			weight := args.Weight * int32(len(args.NodeLabelArgs.PresentLabelsPreference)+len(args.NodeLabelArgs.AbsentLabelsPreference))
 			plugins.Score = appendToPluginSet(plugins.Score, nodelabel.Name, &weight)
 			if args.NodeLabelArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: nodelabel.Name, Args: args.NodeLabelArgs})
 			}
-			return
 		})
 	registry.registerPriorityConfigProducer(serviceaffinity.Name,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, pluginConfig *[]config.PluginConfig) {
 			// If there are n ServiceAffinity priorities in the policy, the weight for the corresponding
 			// score plugin is n*weight (note that the validation logic verifies that all ServiceAffinity
 			// priorities specified in Policy have the same weight).
 			weight := args.Weight * int32(len(args.ServiceAffinityArgs.AntiAffinityLabelsPreference))
 			plugins.Score = appendToPluginSet(plugins.Score, serviceaffinity.Name, &weight)
 			if args.ServiceAffinityArgs != nil {
-				pluginConfig = append(pluginConfig,
+				*pluginConfig = append(*pluginConfig,
 					config.PluginConfig{Name: serviceaffinity.Name, Args: args.ServiceAffinityArgs})
 			}
-			return
 		})
 	registry.registerPriorityConfigProducer(EvenPodsSpreadPriority,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.PreScore = appendToPluginSet(plugins.PreScore, podtopologyspread.Name, nil)
 			plugins.Score = appendToPluginSet(plugins.Score, podtopologyspread.Name, &args.Weight)
-			return
 		})
 
 	return registry
 }
 
+// AppendPredicateConfigs returns predicates configuration that will run as framework plugins.
+// Note that the framework executes plugins according to their order in the Plugins list, and so predicates run as plugins
+// are added to the Plugins list according to the order specified in predicateOrdering.
+func (lr *LegacyRegistry) AppendPredicateConfigs(keys sets.String, args *ConfigProducerArgs, plugins config.Plugins, pluginConfig []config.PluginConfig) (config.Plugins, []config.PluginConfig, error) {
+	allPredicates := keys.Union(lr.mandatoryPredicates)
+
+	// Create the framework plugin configurations, and place them in the order
+	// that the corresponding predicates were supposed to run.
+	for _, predicateKey := range predicateOrdering {
+		if allPredicates.Has(predicateKey) {
+			producer, exist := lr.predicateToConfigProducer[predicateKey]
+			if !exist {
+				return config.Plugins{}, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey)
+			}
+			producer(*args, &plugins, &pluginConfig)
+			allPredicates.Delete(predicateKey)
+		}
+	}
+
+	// Sort the keys so that it is easier for unit tests to do compare.
+	sortedKeys := make([]string, 0, len(allPredicates))
+	for k := range allPredicates {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	for _, predicateKey := range sortedKeys {
+		producer, exist := lr.predicateToConfigProducer[predicateKey]
+		if !exist {
+			return config.Plugins{}, nil, fmt.Errorf("no framework config producer registered for %q", predicateKey)
+		}
+		producer(*args, &plugins, &pluginConfig)
+	}
+
+	return plugins, pluginConfig, nil
+}
+
+// AppendPriorityConfigs returns priorities configuration that will run as framework plugins.
+func (lr *LegacyRegistry) AppendPriorityConfigs(keys map[string]int64, args *ConfigProducerArgs, plugins config.Plugins, pluginConfig []config.PluginConfig) (config.Plugins, []config.PluginConfig, error) {
+	// Sort the keys so that it is easier for unit tests to do compare.
+	sortedKeys := make([]string, 0, len(keys))
+	for k := range keys {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	for _, priority := range sortedKeys {
+		weight := keys[priority]
+		producer, exist := lr.priorityToConfigProducer[priority]
+		if !exist {
+			return config.Plugins{}, nil, fmt.Errorf("no config producer registered for %q", priority)
+		}
+		a := *args
+		a.Weight = int32(weight)
+		producer(a, &plugins, &pluginConfig)
+	}
+	return plugins, pluginConfig, nil
+}
+
 // registers a config producer for a predicate.
-func (lr *LegacyRegistry) registerPredicateConfigProducer(name string, producer ConfigProducer) {
+func (lr *LegacyRegistry) registerPredicateConfigProducer(name string, producer configProducer) {
-	if _, exist := lr.PredicateToConfigProducer[name]; exist {
+	if _, exist := lr.predicateToConfigProducer[name]; exist {
 		klog.Fatalf("already registered %q", name)
 	}
-	lr.PredicateToConfigProducer[name] = producer
+	lr.predicateToConfigProducer[name] = producer
 }
 
 // registers a framework config producer for a priority.
-func (lr *LegacyRegistry) registerPriorityConfigProducer(name string, producer ConfigProducer) {
+func (lr *LegacyRegistry) registerPriorityConfigProducer(name string, producer configProducer) {
-	if _, exist := lr.PriorityToConfigProducer[name]; exist {
+	if _, exist := lr.priorityToConfigProducer[name]; exist {
 		klog.Fatalf("already registered %q", name)
 	}
-	lr.PriorityToConfigProducer[name] = producer
+	lr.priorityToConfigProducer[name] = producer
 }
 
 func appendToPluginSet(set *config.PluginSet, name string, weight *int32) *config.PluginSet {
@@ -479,7 +508,7 @@ func (lr *LegacyRegistry) ProcessPredicatePolicy(policy config.PredicatePolicy,
 		predicateName = PodFitsHostPortsPred
 	}
 
-	if _, ok := lr.PredicateToConfigProducer[predicateName]; ok {
+	if _, ok := lr.predicateToConfigProducer[predicateName]; ok {
 		// checking to see if a pre-defined predicate is requested
 		klog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name)
 		return predicateName
@@ -535,7 +564,7 @@ func (lr *LegacyRegistry) ProcessPriorityPolicy(policy config.PriorityPolicy, co
 		priorityName = SelectorSpreadPriority
 	}
 
-	if _, ok := lr.PriorityToConfigProducer[priorityName]; ok {
+	if _, ok := lr.priorityToConfigProducer[priorityName]; ok {
 		klog.V(2).Infof("Priority type %s already registered, reusing.", priorityName)
 		return priorityName
 	}

pkg/scheduler/framework/plugins/legacy_registry_test.go

@@ -17,104 +17,76 @@ limitations under the License.
 package plugins
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
 )
 
-func produceConfig(keys []string, producersMap map[string]ConfigProducer, args ConfigProducerArgs) (*config.Plugins, []config.PluginConfig, error) {
-	var plugins config.Plugins
-	var pluginConfig []config.PluginConfig
-	for _, k := range keys {
-		p, exist := producersMap[k]
-		if !exist {
-			return nil, nil, fmt.Errorf("finding key %q", k)
-		}
-		pl, plc := p(args)
-		plugins.Append(&pl)
-		pluginConfig = append(pluginConfig, plc...)
-	}
-	return &plugins, pluginConfig, nil
-}
-
 func TestRegisterConfigProducers(t *testing.T) {
 	registry := NewLegacyRegistry()
 	testPredicateName1 := "testPredicate1"
 	testFilterName1 := "testFilter1"
 	registry.registerPredicateConfigProducer(testPredicateName1,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName1, nil)
-			return
 		})
 
 	testPredicateName2 := "testPredicate2"
 	testFilterName2 := "testFilter2"
 	registry.registerPredicateConfigProducer(testPredicateName2,
-		func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(_ ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Filter = appendToPluginSet(plugins.Filter, testFilterName2, nil)
-			return
 		})
 
 	testPriorityName1 := "testPriority1"
 	testScoreName1 := "testScore1"
 	registry.registerPriorityConfigProducer(testPriorityName1,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, testScoreName1, &args.Weight)
-			return
 		})
 
 	testPriorityName2 := "testPriority2"
 	testScoreName2 := "testScore2"
 	registry.registerPriorityConfigProducer(testPriorityName2,
-		func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
+		func(args ConfigProducerArgs, plugins *config.Plugins, _ *[]config.PluginConfig) {
 			plugins.Score = appendToPluginSet(plugins.Score, testScoreName2, &args.Weight)
-			return
 		})
 
 	args := ConfigProducerArgs{Weight: 1}
-	predicatePlugins, _, err := produceConfig(
-		[]string{testPredicateName1, testPredicateName2}, registry.PredicateToConfigProducer, args)
-	if err != nil {
-		t.Fatalf("producing predicate framework configs: %v.", err)
-	}
-
-	priorityPlugins, _, err := produceConfig(
-		[]string{testPriorityName1, testPriorityName2}, registry.PriorityToConfigProducer, args)
-	if err != nil {
-		t.Fatalf("producing predicate framework configs: %v.", err)
-	}
-
-	// Verify that predicates and priorities are in the map and produce the expected score configurations.
 	var gotPlugins config.Plugins
-	gotPlugins.Append(predicatePlugins)
+	gotPlugins, _, err := registry.AppendPredicateConfigs(sets.NewString(testPredicateName1, testPredicateName2), &args, gotPlugins, nil)
-	gotPlugins.Append(priorityPlugins)
+	if err != nil {
+		t.Fatalf("producing predicate framework configs: %v.", err)
+	}
+
+	priorities := map[string]int64{
+		testPriorityName1: 1,
+		testPriorityName2: 1,
+	}
+	gotPlugins, _, err = registry.AppendPriorityConfigs(priorities, &args, gotPlugins, nil)
+	if err != nil {
+		t.Fatalf("producing priority framework configs: %v.", err)
+	}
 
-	// Verify the aggregated configuration.
 	wantPlugins := config.Plugins{
-		QueueSort: &config.PluginSet{},
-		PreFilter: &config.PluginSet{},
 		Filter: &config.PluginSet{
 			Enabled: []config.Plugin{
+				{Name: nodeunschedulable.Name},
+				{Name: tainttoleration.Name},
 				{Name: testFilterName1},
 				{Name: testFilterName2},
 			},
 		},
-		PostFilter: &config.PluginSet{},
-		PreScore:   &config.PluginSet{},
 		Score: &config.PluginSet{
 			Enabled: []config.Plugin{
 				{Name: testScoreName1, Weight: 1},
 				{Name: testScoreName2, Weight: 1},
 			},
 		},
-		Reserve:  &config.PluginSet{},
-		Permit:   &config.PluginSet{},
-		PreBind:  &config.PluginSet{},
-		Bind:     &config.PluginSet{},
-		PostBind: &config.PluginSet{},
 	}
 
 	if diff := cmp.Diff(wantPlugins, gotPlugins); diff != "" {