Implement multi-scheduler:

1. Name default scheduler with name `kube-scheduler` 2. The default scheduler only schedules the pods meeting the following condition: - the pod has no annotation "scheduler.alpha.kubernetes.io/name: <scheduler-name>" - the pod has annotation "scheduler.alpha.kubernetes.io/name: kube-scheduler" update gofmt update according to @david's review run hack/test-integration.sh, hack/test-go.sh and local e2e.test
2015-11-27 17:07:17 +08:00
parent 4b18fa553f
commit d9f3607292
13 changed files with 302 additions and 19 deletions
--- a/test/integration/scheduler_test.go
+++ b/test/integration/scheduler_test.go
@@ -65,13 +65,13 @@ func TestUnschedulableNodes(t *testing.T) {

 	restClient := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})

-	schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
+	schedulerConfigFactory := factory.NewConfigFactory(restClient, nil, api.DefaultSchedulerName)
 	schedulerConfig, err := schedulerConfigFactory.Create()
 	if err != nil {
 		t.Fatalf("Couldn't create scheduler config: %v", err)
 	}
 	eventBroadcaster := record.NewBroadcaster()
-	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
+	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
 	eventBroadcaster.StartRecordingToSink(restClient.Events(""))
 	scheduler.New(schedulerConfig).Run()

@@ -273,3 +273,177 @@ func DoTestUnschedulableNodes(t *testing.T, restClient *client.Client, nodeStore
 		}
 	}
 }
+
+func TestMultiScheduler(t *testing.T) {
+	framework.DeleteAllEtcdKeys()
+
+	var m *master.Master
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		m.Handler.ServeHTTP(w, req)
+	}))
+	defer s.Close()
+
+	masterConfig := framework.NewIntegrationTestMasterConfig()
+	m = master.New(masterConfig)
+	/*
+		This integration tests the multi-scheduler feature in the following way:
+		1. create a default scheduler
+		2. create a node
+		3. create 3 pods: testPodNoAnnotation, testPodWithAnnotationFitsDefault and testPodWithAnnotationFitsFoo
+			- note: the first two should be picked and scheduled by default scheduler while the last one should be
+			        picked by scheduler of name "foo-scheduler" which does not exist yet.
+		4. **check point-1**:
+			- testPodNoAnnotation, testPodWithAnnotationFitsDefault should be scheduled
+			- testPodWithAnnotationFitsFoo should NOT be scheduled
+		5. create a scheduler with name "foo-scheduler"
+		6. **check point-2**:
+			- testPodWithAnnotationFitsFoo should be scheduled
+		7. stop default scheduler
+		8. create 2 pods: testPodNoAnnotation2 and testPodWithAnnotationFitsDefault2
+			- note: these two pods belong to default scheduler which no longer exists
+		9. **check point-3**:
+			- testPodNoAnnotation2 and testPodWithAnnotationFitsDefault2 shoule NOT be scheduled
+	*/
+	// 1. create and start default-scheduler
+	restClient := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
+
+	schedulerConfigFactory := factory.NewConfigFactory(restClient, nil, api.DefaultSchedulerName)
+	schedulerConfig, err := schedulerConfigFactory.Create()
+	if err != nil {
+		t.Fatalf("Couldn't create scheduler config: %v", err)
+	}
+	eventBroadcaster := record.NewBroadcaster()
+	schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
+	eventBroadcaster.StartRecordingToSink(restClient.Events(""))
+	scheduler.New(schedulerConfig).Run()
+	// default-scheduler will be stopped later
+
+	// 2. create a node
+	node := &api.Node{
+		ObjectMeta: api.ObjectMeta{Name: "node-multi-scheduler-test-node"},
+		Spec:       api.NodeSpec{Unschedulable: false},
+		Status: api.NodeStatus{
+			Capacity: api.ResourceList{
+				api.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
+			},
+		},
+	}
+	restClient.Nodes().Create(node)
+
+	// 3. create 3 pods for testing
+	podWithNoAnnotation := createPod("pod-with-no-annotation", nil)
+	testPodNoAnnotation, err := restClient.Pods(api.NamespaceDefault).Create(podWithNoAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to create pod: %v", err)
+	}
+
+	schedulerAnnotationFitsDefault := map[string]string{"scheduler.alpha.kubernetes.io/name": "default-scheduler"}
+	podWithAnnotationFitsDefault := createPod("pod-with-annotation-fits-default", schedulerAnnotationFitsDefault)
+	testPodWithAnnotationFitsDefault, err := restClient.Pods(api.NamespaceDefault).Create(podWithAnnotationFitsDefault)
+	if err != nil {
+		t.Fatalf("Failed to create pod: %v", err)
+	}
+
+	schedulerAnnotationFitsFoo := map[string]string{"scheduler.alpha.kubernetes.io/name": "foo-scheduler"}
+	podWithAnnotationFitsFoo := createPod("pod-with-annotation-fits-foo", schedulerAnnotationFitsFoo)
+	testPodWithAnnotationFitsFoo, err := restClient.Pods(api.NamespaceDefault).Create(podWithAnnotationFitsFoo)
+	if err != nil {
+		t.Fatalf("Failed to create pod: %v", err)
+	}
+
+	// 4. **check point-1**:
+	//		- testPodNoAnnotation, testPodWithAnnotationFitsDefault should be scheduled
+	//		- testPodWithAnnotationFitsFoo should NOT be scheduled
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodNoAnnotation.Namespace, testPodNoAnnotation.Name))
+	if err != nil {
+		t.Errorf("Test MultiScheduler: %s Pod not scheduled: %v", testPodNoAnnotation.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod scheduled", testPodNoAnnotation.Name)
+	}
+
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodWithAnnotationFitsDefault.Namespace, testPodWithAnnotationFitsDefault.Name))
+	if err != nil {
+		t.Errorf("Test MultiScheduler: %s Pod not scheduled: %v", testPodWithAnnotationFitsDefault.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod scheduled", testPodWithAnnotationFitsDefault.Name)
+	}
+
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodWithAnnotationFitsFoo.Namespace, testPodWithAnnotationFitsFoo.Name))
+	if err == nil {
+		t.Errorf("Test MultiScheduler: %s Pod got scheduled, %v", testPodWithAnnotationFitsFoo.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod not scheduled", testPodWithAnnotationFitsFoo.Name)
+	}
+
+	// 5. create and start a scheduler with name "foo-scheduler"
+	restClient2 := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
+
+	schedulerConfigFactory2 := factory.NewConfigFactory(restClient2, nil, "foo-scheduler")
+	schedulerConfig2, err := schedulerConfigFactory2.Create()
+	if err != nil {
+		t.Errorf("Couldn't create scheduler config: %v", err)
+	}
+	eventBroadcaster2 := record.NewBroadcaster()
+	schedulerConfig2.Recorder = eventBroadcaster2.NewRecorder(api.EventSource{Component: "foo-scheduler"})
+	eventBroadcaster2.StartRecordingToSink(restClient2.Events(""))
+	scheduler.New(schedulerConfig2).Run()
+
+	defer close(schedulerConfig2.StopEverything)
+
+	//	6. **check point-2**:
+	//		- testPodWithAnnotationFitsFoo should be scheduled
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodWithAnnotationFitsFoo.Namespace, testPodWithAnnotationFitsFoo.Name))
+	if err != nil {
+		t.Errorf("Test MultiScheduler: %s Pod not scheduled, %v", testPodWithAnnotationFitsFoo.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod scheduled", testPodWithAnnotationFitsFoo.Name)
+	}
+
+	//	7. delete the pods that were scheduled by the default scheduler, and stop the default scheduler
+	err = restClient.Pods(api.NamespaceDefault).Delete(testPodNoAnnotation.Name, api.NewDeleteOptions(0))
+	if err != nil {
+		t.Errorf("Failed to delete pod: %v", err)
+	}
+	err = restClient.Pods(api.NamespaceDefault).Delete(testPodWithAnnotationFitsDefault.Name, api.NewDeleteOptions(0))
+	if err != nil {
+		t.Errorf("Failed to delete pod: %v", err)
+	}
+	close(schedulerConfig.StopEverything)
+
+	//	8. create 2 pods: testPodNoAnnotation2 and testPodWithAnnotationFitsDefault2
+	//		- note: these two pods belong to default scheduler which no longer exists
+	podWithNoAnnotation2 := createPod("pod-with-no-annotation2", nil)
+	podWithAnnotationFitsDefault2 := createPod("pod-with-annotation-fits-default2", schedulerAnnotationFitsDefault)
+	testPodNoAnnotation2, err := restClient.Pods(api.NamespaceDefault).Create(podWithNoAnnotation2)
+	if err != nil {
+		t.Fatalf("Failed to create pod: %v", err)
+	}
+	testPodWithAnnotationFitsDefault2, err := restClient.Pods(api.NamespaceDefault).Create(podWithAnnotationFitsDefault2)
+	if err != nil {
+		t.Fatalf("Failed to create pod: %v", err)
+	}
+
+	//	9. **check point-3**:
+	//		- testPodNoAnnotation2 and testPodWithAnnotationFitsDefault2 shoule NOT be scheduled
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodNoAnnotation2.Namespace, testPodNoAnnotation2.Name))
+	if err == nil {
+		t.Errorf("Test MultiScheduler: %s Pod got scheduled, %v", testPodNoAnnotation2.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod not scheduled", testPodNoAnnotation2.Name)
+	}
+	err = wait.Poll(time.Second, time.Second*5, podScheduled(restClient, testPodWithAnnotationFitsDefault2.Namespace, testPodWithAnnotationFitsDefault2.Name))
+	if err == nil {
+		t.Errorf("Test MultiScheduler: %s Pod got scheduled, %v", testPodWithAnnotationFitsDefault2.Name, err)
+	} else {
+		t.Logf("Test MultiScheduler: %s Pod scheduled", testPodWithAnnotationFitsDefault2.Name)
+	}
+}
+
+func createPod(name string, annotation map[string]string) *api.Pod {
+	return &api.Pod{
+		ObjectMeta: api.ObjectMeta{Name: name, Annotations: annotation},
+		Spec: api.PodSpec{
+			Containers: []api.Container{{Name: "container", Image: "kubernetes/pause:go"}},
+		},
+	}
+}