Capture defaulted plugin configs from framework

Some plugin configs are not produced until the framework is instantiated. Add a callback to capture them inside the framework constructor.

Change-Id: Id3f709b6461ccd0eafec7d21412cda093d9c4645

pkg/scheduler/BUILD

@@ -66,6 +66,7 @@ go_test(
         "//pkg/scheduler/framework/plugins:go_default_library",
         "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library",
         "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library",
+        "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library",
         "//pkg/scheduler/framework/plugins/nodelabel:go_default_library",
         "//pkg/scheduler/framework/plugins/nodeports:go_default_library",
         "//pkg/scheduler/framework/plugins/noderesources:go_default_library",

pkg/scheduler/apis/config/testing/compatibility_test.go

@@ -1578,6 +1578,48 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 		"PreBindPlugin": {{Name: "VolumeBinding"}},
 		"BindPlugin":    {{Name: "DefaultBinder"}},
 	}
+	defaultPluginConfigs := []config.PluginConfig{
+		{
+			Name: "DefaultPreemption",
+			Args: &config.DefaultPreemptionArgs{
+				MinCandidateNodesPercentage: 10,
+				MinCandidateNodesAbsolute:   100,
+			},
+		},
+		{
+			Name: "InterPodAffinity",
+			Args: &config.InterPodAffinityArgs{
+				HardPodAffinityWeight: 1,
+			},
+		},
+		{
+			Name: "NodeAffinity",
+			Args: &config.NodeAffinityArgs{},
+		},
+		{
+			Name: "NodeResourcesFit",
+			Args: &config.NodeResourcesFitArgs{},
+		},
+		{
+			Name: "NodeResourcesLeastAllocated",
+			Args: &config.NodeResourcesLeastAllocatedArgs{
+				Resources: []config.ResourceSpec{
+					{Name: "cpu", Weight: 1},
+					{Name: "memory", Weight: 1},
+				},
+			},
+		},
+		{
+			Name: "PodTopologySpread",
+			Args: &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting},
+		},
+		{
+			Name: "VolumeBinding",
+			Args: &config.VolumeBindingArgs{
+				BindTimeoutSeconds: 600,
+			},
+		},
+	}
 
 	testcases := []struct {
 		name             string
@@ -1589,11 +1631,76 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 		{
 			name:             "default plugins",
 			wantPlugins:      defaultPlugins,
-			wantPluginConfig: nil,
+			wantPluginConfig: defaultPluginConfigs,
 		},
 		{
-			name:        "default plugins with customized plugin config",
+			name: "in-tree plugins with customized plugin config",
-			wantPlugins: defaultPlugins,
+			plugins: config.Plugins{
+				Filter: &config.PluginSet{
+					Enabled: []config.Plugin{
+						{Name: "NodeLabel"},
+						{Name: "ServiceAffinity"},
+					},
+				},
+				Score: &config.PluginSet{
+					Enabled: []config.Plugin{
+						{Name: "RequestedToCapacityRatio"},
+					},
+				},
+			},
+			wantPlugins: map[string][]config.Plugin{
+				"QueueSortPlugin": {
+					{Name: "PrioritySort"},
+				},
+				"PreFilterPlugin": {
+					{Name: "NodeResourcesFit"},
+					{Name: "NodePorts"},
+					{Name: "PodTopologySpread"},
+					{Name: "InterPodAffinity"},
+					{Name: "VolumeBinding"},
+				},
+				"FilterPlugin": {
+					{Name: "NodeUnschedulable"},
+					{Name: "NodeName"},
+					{Name: "TaintToleration"},
+					{Name: "NodeAffinity"},
+					{Name: "NodePorts"},
+					{Name: "NodeResourcesFit"},
+					{Name: "VolumeRestrictions"},
+					{Name: "EBSLimits"},
+					{Name: "GCEPDLimits"},
+					{Name: "NodeVolumeLimits"},
+					{Name: "AzureDiskLimits"},
+					{Name: "VolumeBinding"},
+					{Name: "VolumeZone"},
+					{Name: "PodTopologySpread"},
+					{Name: "InterPodAffinity"},
+					{Name: "NodeLabel"},
+					{Name: "ServiceAffinity"},
+				},
+				"PostFilterPlugin": {
+					{Name: "DefaultPreemption"},
+				},
+				"PreScorePlugin": {
+					{Name: "InterPodAffinity"},
+					{Name: "PodTopologySpread"},
+					{Name: "TaintToleration"},
+				},
+				"ScorePlugin": {
+					{Name: "NodeResourcesBalancedAllocation", Weight: 1},
+					{Name: "ImageLocality", Weight: 1},
+					{Name: "InterPodAffinity", Weight: 1},
+					{Name: "NodeResourcesLeastAllocated", Weight: 1},
+					{Name: "NodeAffinity", Weight: 1},
+					{Name: "NodePreferAvoidPods", Weight: 10000},
+					{Name: "PodTopologySpread", Weight: 2},
+					{Name: "TaintToleration", Weight: 1},
+					{Name: "RequestedToCapacityRatio", Weight: 1},
+				},
+				"ReservePlugin": {{Name: "VolumeBinding"}},
+				"PreBindPlugin": {{Name: "VolumeBinding"}},
+				"BindPlugin":    {{Name: "DefaultBinder"}},
+			},
 			pluginConfig: []config.PluginConfig{
 				{
 					Name: "NodeResourcesFit",
@@ -1660,12 +1767,47 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				},
 			},
 			wantPluginConfig: []config.PluginConfig{
+				{
+					Name: "DefaultPreemption",
+					Args: &config.DefaultPreemptionArgs{
+						MinCandidateNodesPercentage: 10,
+						MinCandidateNodesAbsolute:   100,
+					},
+				},
+				{
+					Name: "InterPodAffinity",
+					Args: &config.InterPodAffinityArgs{
+						HardPodAffinityWeight: 100,
+					},
+				},
+				{
+					Name: "NodeAffinity",
+					Args: &config.NodeAffinityArgs{},
+				},
+				{
+					Name: "NodeLabel",
+					Args: &config.NodeLabelArgs{
+						PresentLabels:           []string{"foo", "bar"},
+						AbsentLabels:            []string{"apple"},
+						PresentLabelsPreference: []string{"dog"},
+						AbsentLabelsPreference:  []string{"cat"},
+					},
+				},
 				{
 					Name: "NodeResourcesFit",
 					Args: &config.NodeResourcesFitArgs{
 						IgnoredResources: []string{"foo", "bar"},
 					},
 				},
+				{
+					Name: "NodeResourcesLeastAllocated",
+					Args: &config.NodeResourcesLeastAllocatedArgs{
+						Resources: []config.ResourceSpec{
+							{Name: "cpu", Weight: 1},
+							{Name: "memory", Weight: 1},
+						},
+					},
+				},
 				{
 					Name: "PodTopologySpread",
 					Args: &config.PodTopologySpreadArgs{
@@ -1695,21 +1837,6 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 						},
 					},
 				},
-				{
-					Name: "InterPodAffinity",
-					Args: &config.InterPodAffinityArgs{
-						HardPodAffinityWeight: 100,
-					},
-				},
-				{
-					Name: "NodeLabel",
-					Args: &config.NodeLabelArgs{
-						PresentLabels:           []string{"foo", "bar"},
-						AbsentLabels:            []string{"apple"},
-						PresentLabelsPreference: []string{"dog"},
-						AbsentLabelsPreference:  []string{"cat"},
-					},
-				},
 				{
 					Name: "ServiceAffinity",
 					Args: &config.ServiceAffinityArgs{
@@ -1849,6 +1976,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				},
 				PreScore: &config.PluginSet{
 					Enabled: []config.Plugin{
+						{Name: "PodTopologySpread"},
 						{Name: "TaintToleration"},
 						{Name: "SelectorSpread"},
 						{Name: "InterPodAffinity"},
@@ -1859,6 +1987,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				},
 				Score: &config.PluginSet{
 					Enabled: []config.Plugin{
+						{Name: "PodTopologySpread", Weight: 24},
 						{Name: "TaintToleration", Weight: 24},
 						{Name: "SelectorSpread", Weight: 24},
 						{Name: "NodePreferAvoidPods", Weight: 24},
@@ -1906,11 +2035,13 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 					{Name: "DefaultPreemption"},
 				},
 				"PreScorePlugin": {
+					{Name: "PodTopologySpread"},
 					{Name: "TaintToleration"},
 					{Name: "SelectorSpread"},
 					{Name: "InterPodAffinity"},
 				},
 				"ScorePlugin": {
+					{Name: "PodTopologySpread", Weight: 24},
 					{Name: "TaintToleration", Weight: 24},
 					{Name: "SelectorSpread", Weight: 24},
 					{Name: "NodePreferAvoidPods", Weight: 24},
@@ -1924,7 +2055,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				"PreBindPlugin": {{Name: "VolumeBinding"}},
 				"BindPlugin":    {{Name: "DefaultBinder"}},
 			},
-			wantPluginConfig: nil,
+			wantPluginConfig: defaultPluginConfigs,
 		},
 	}
 	for _, tc := range testcases {

pkg/scheduler/factory.go

@@ -137,12 +137,8 @@ func (c *Configurator) create() (*Scheduler, error) {
 		frameworkruntime.WithSnapshotSharedLister(c.nodeInfoSnapshot),
 		frameworkruntime.WithRunAllFilters(c.alwaysCheckAllPredicates),
 		frameworkruntime.WithPodNominator(nominator),
+		frameworkruntime.WithCaptureProfile(frameworkruntime.CaptureProfile(c.frameworkCapturer)),
 	)
-	for _, p := range c.profiles {
-		if c.frameworkCapturer != nil {
-			c.frameworkCapturer(p)
-		}
-	}
 	if err != nil {
 		return nil, fmt.Errorf("initializing profiles: %v", err)
 	}

pkg/scheduler/factory_test.go

@@ -39,10 +39,12 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/framework"
 	frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodelabel"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
 	frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
 	internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
 	internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
@@ -105,10 +107,39 @@ func TestCreateFromConfig(t *testing.T) {
 				"apiVersion" : "v1"
 			}`),
 			wantPluginConfig: []schedulerapi.PluginConfig{
+				{
+					Name: interpodaffinity.Name,
+					Args: &schedulerapi.InterPodAffinityArgs{
+						HardPodAffinityWeight: 1,
+					},
+				},
+				{
+					Name: nodeaffinity.Name,
+					Args: &schedulerapi.NodeAffinityArgs{},
+				},
+				{
+					Name: noderesources.FitName,
+					Args: &schedulerapi.NodeResourcesFitArgs{},
+				},
+				{
+					Name: noderesources.LeastAllocatedName,
+					Args: &schedulerapi.NodeResourcesLeastAllocatedArgs{
+						Resources: []schedulerapi.ResourceSpec{
+							{Name: "cpu", Weight: 1},
+							{Name: "memory", Weight: 1},
+						},
+					},
+				},
 				{
 					Name: podtopologyspread.Name,
 					Args: &schedulerapi.PodTopologySpreadArgs{DefaultingType: schedulerapi.SystemDefaulting},
 				},
+				{
+					Name: volumebinding.Name,
+					Args: &schedulerapi.VolumeBindingArgs{
+						BindTimeoutSeconds: 600,
+					},
+				},
 			},
 			wantPlugins: &schedulerapi.Plugins{
 				QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@@ -200,6 +231,16 @@ func TestCreateFromConfig(t *testing.T) {
 				]
 			}`),
 			wantPluginConfig: []schedulerapi.PluginConfig{
+				{
+					Name: interpodaffinity.Name,
+					Args: &schedulerapi.InterPodAffinityArgs{
+						HardPodAffinityWeight: 1,
+					},
+				},
+				{
+					Name: nodeaffinity.Name,
+					Args: &schedulerapi.NodeAffinityArgs{},
+				},
 				{
 					Name: nodelabel.Name,
 					Args: &schedulerapi.NodeLabelArgs{
@@ -209,13 +250,6 @@ func TestCreateFromConfig(t *testing.T) {
 						AbsentLabelsPreference:  []string{"l2"},
 					},
 				},
-				{
-					Name: serviceaffinity.Name,
-					Args: &schedulerapi.ServiceAffinityArgs{
-						AffinityLabels:               []string{"zone", "foo"},
-						AntiAffinityLabelsPreference: []string{"rack", "zone"},
-					},
-				},
 				{
 					Name: noderesources.RequestedToCapacityRatioName,
 					Args: &schedulerapi.RequestedToCapacityRatioArgs{
@@ -226,6 +260,13 @@ func TestCreateFromConfig(t *testing.T) {
 						Resources: []schedulerapi.ResourceSpec{},
 					},
 				},
+				{
+					Name: serviceaffinity.Name,
+					Args: &schedulerapi.ServiceAffinityArgs{
+						AffinityLabels:               []string{"zone", "foo"},
+						AntiAffinityLabelsPreference: []string{"rack", "zone"},
+					},
+				},
 			},
 			wantPlugins: &schedulerapi.Plugins{
 				QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
@@ -262,46 +303,31 @@ func TestCreateFromConfig(t *testing.T) {
 				"kind" : "Policy",
 				"apiVersion" : "v1",
 				"predicates" : [
-					{"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}},
-					{"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}},
 					{"name" : "PodFitsResources"},
 					{"name" : "PodFitsHostPorts"}
 				],
 				"priorities" : [
-					{"name" : "RackSpread", "weight" : 3, "argument" : {"serviceAntiAffinity" : {"label" : "rack"}}},
-					{"name" : "NodeAffinityPriority", "weight" : 2},
-					{"name" : "ImageLocalityPriority", "weight" : 1},
 					{"name" : "InterPodAffinityPriority", "weight" : 1}
 				],
 				"hardPodAffinitySymmetricWeight" : 10
 			}`),
 			wantPluginConfig: []schedulerapi.PluginConfig{
-				{
-					Name: nodelabel.Name,
-					Args: &schedulerapi.NodeLabelArgs{
-						PresentLabels: []string{"zone"},
-					},
-				},
-				{
-					Name: serviceaffinity.Name,
-					Args: &schedulerapi.ServiceAffinityArgs{
-						AffinityLabels:               []string{"zone"},
-						AntiAffinityLabelsPreference: []string{"rack"},
-					},
-				},
 				{
 					Name: interpodaffinity.Name,
 					Args: &schedulerapi.InterPodAffinityArgs{
 						HardPodAffinityWeight: 10,
 					},
 				},
+				{
+					Name: "NodeResourcesFit",
+					Args: &schedulerapi.NodeResourcesFitArgs{},
+				},
 			},
 			wantPlugins: &schedulerapi.Plugins{
 				QueueSort: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "PrioritySort"}}},
 				PreFilter: &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{
 					{Name: "NodePorts"},
 					{Name: "NodeResourcesFit"},
-					{Name: "ServiceAffinity"},
 				}},
 				Filter: &schedulerapi.PluginSet{
 					Enabled: []schedulerapi.Plugin{
@@ -309,18 +335,13 @@ func TestCreateFromConfig(t *testing.T) {
 						{Name: "NodePorts"},
 						{Name: "NodeResourcesFit"},
 						{Name: "TaintToleration"},
-						{Name: "NodeLabel"},
-						{Name: "ServiceAffinity"},
 					},
 				},
 				PostFilter: &schedulerapi.PluginSet{},
 				PreScore:   &schedulerapi.PluginSet{Enabled: []schedulerapi.Plugin{{Name: "InterPodAffinity"}}},
 				Score: &schedulerapi.PluginSet{
 					Enabled: []schedulerapi.Plugin{
-						{Name: "ImageLocality", Weight: 1},
 						{Name: "InterPodAffinity", Weight: 1},
-						{Name: "NodeAffinity", Weight: 2},
-						{Name: "ServiceAffinity", Weight: 3},
 					},
 				},
 				Reserve:  &schedulerapi.PluginSet{},

pkg/scheduler/framework/runtime/framework.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"sort"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
@@ -130,6 +131,7 @@ type frameworkOptions struct {
 	podNominator         framework.PodNominator
 	extenders            []framework.Extender
 	runAllFilters        bool
+	captureProfile       CaptureProfile
 }
 
 // Option for the frameworkImpl.
@@ -199,6 +201,16 @@ func WithExtenders(extenders []framework.Extender) Option {
 	}
 }
 
+// CaptureProfile is a callback to capture a finalized profile.
+type CaptureProfile func(config.KubeSchedulerProfile)
+
+// WithCaptureProfile sets a callback to capture the finalized profile.
+func WithCaptureProfile(c CaptureProfile) Option {
+	return func(o *frameworkOptions) {
+		o.captureProfile = c
+	}
+}
+
 var defaultFrameworkOptions = frameworkOptions{
 	metricsRecorder: newMetricsRecorder(1000, time.Second),
 }
@@ -258,6 +270,11 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
 		}
 		pluginConfig[name] = args[i].Args
 	}
+	outputProfile := config.KubeSchedulerProfile{
+		SchedulerName: f.profileName,
+		Plugins:       plugins,
+		PluginConfig:  make([]config.PluginConfig, 0, len(pg)),
+	}
 
 	pluginsMap := make(map[string]framework.Plugin)
 	var totalPriority int64
@@ -271,6 +288,12 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
 		if err != nil {
 			return nil, fmt.Errorf("getting args for Plugin %q: %w", name, err)
 		}
+		if args != nil {
+			outputProfile.PluginConfig = append(outputProfile.PluginConfig, config.PluginConfig{
+				Name: name,
+				Args: args,
+			})
+		}
 		p, err := factory(args, f)
 		if err != nil {
 			return nil, fmt.Errorf("initializing plugin %q: %w", name, err)
@@ -314,6 +337,17 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi
 		return nil, fmt.Errorf("at least one bind plugin is needed")
 	}
 
+	if options.captureProfile != nil {
+		if len(outputProfile.PluginConfig) != 0 {
+			sort.Slice(outputProfile.PluginConfig, func(i, j int) bool {
+				return outputProfile.PluginConfig[i].Name < outputProfile.PluginConfig[j].Name
+			})
+		} else {
+			outputProfile.PluginConfig = nil
+		}
+		options.captureProfile(outputProfile)
+	}
+
 	return f, nil
 }