Teardown only scheduler in integration tests

Signed-off-by: Kante Yin <kerthcet@gmail.com>

test/integration/util/util.go

@@ -143,15 +143,24 @@ func StartFakePVController(ctx context.Context, clientSet clientset.Interface) {
 
 // TestContext store necessary context info
 type TestContext struct {
-	CloseFn            framework.TearDownFunc
 	NS                 *v1.Namespace
 	ClientSet          clientset.Interface
 	KubeConfig         *restclient.Config
 	InformerFactory    informers.SharedInformerFactory
 	DynInformerFactory dynamicinformer.DynamicSharedInformerFactory
 	Scheduler          *scheduler.Scheduler
-	Ctx                context.Context
-	CancelFn           context.CancelFunc
+	// This is the top context when initializing the test environment.
+	Ctx context.Context
+	// CancelFn will cancel the context above.
+	CancelFn context.CancelFunc
+	// CloseFn will stop the apiserver and clean up the resources
+	// after itself, including shutting down its storage layer.
+	CloseFn framework.TearDownFunc
+	// This is the context when initializing scheduler.
+	SchedulerCtx context.Context
+	// SchedulerCloseFn will tear down the resources in creating scheduler,
+	// including the scheduler itself.
+	SchedulerCloseFn framework.TearDownFunc
 }
 
 // CleanupNodes cleans all nodes which were created during integration test
@@ -176,25 +185,39 @@ func PodDeleted(c clientset.Interface, podNamespace, podName string) wait.Condit
 	}
 }
 
-// SyncInformerFactory starts informer and waits for caches to be synced
-func SyncInformerFactory(testCtx *TestContext) {
-	testCtx.InformerFactory.Start(testCtx.Ctx.Done())
-	if testCtx.DynInformerFactory != nil {
-		testCtx.DynInformerFactory.Start(testCtx.Ctx.Done())
+// PodsCleanedUp returns true if all pods are deleted in the specific namespace.
+func PodsCleanedUp(ctx context.Context, c clientset.Interface, namespace string) wait.ConditionFunc {
+	return func() (bool, error) {
+		list, err := c.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		return len(list.Items) == 0, nil
 	}
-	testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done())
+}
+
+// SyncSchedulerInformerFactory starts informer and waits for caches to be synced
+func SyncSchedulerInformerFactory(testCtx *TestContext) {
+	testCtx.InformerFactory.Start(testCtx.SchedulerCtx.Done())
 	if testCtx.DynInformerFactory != nil {
-		testCtx.DynInformerFactory.WaitForCacheSync(testCtx.Ctx.Done())
+		testCtx.DynInformerFactory.Start(testCtx.SchedulerCtx.Done())
+	}
+	testCtx.InformerFactory.WaitForCacheSync(testCtx.SchedulerCtx.Done())
+	if testCtx.DynInformerFactory != nil {
+		testCtx.DynInformerFactory.WaitForCacheSync(testCtx.SchedulerCtx.Done())
 	}
 }
 
 // CleanupTest cleans related resources which were created during integration test
 func CleanupTest(t *testing.T, testCtx *TestContext) {
-	// Kill the scheduler.
+	// Cancel the context of the whole test environment, it will terminate the scheduler as well.
 	testCtx.CancelFn()
-	// Cleanup nodes.
-	testCtx.ClientSet.CoreV1().Nodes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
+
+	// Cleanup nodes and namespaces.
+	testCtx.ClientSet.CoreV1().Nodes().DeleteCollection(testCtx.Ctx, *metav1.NewDeleteOptions(0), metav1.ListOptions{})
 	framework.DeleteNamespaceOrDie(testCtx.ClientSet, testCtx.NS, t)
+
+	// Terminate the apiserver.
 	testCtx.CloseFn()
 }
 
@@ -330,11 +353,13 @@ func UpdateNodeStatus(cs clientset.Interface, node *v1.Node) error {
 
 // InitTestAPIServer initializes a test environment and creates an API server with default
 // configuration.
+// It registers cleanup functions to t.Cleanup(), they will be called when the test completes,
+// no need to do this again.
 func InitTestAPIServer(t *testing.T, nsPrefix string, admission admission.Interface) *TestContext {
-	ctx, cancelFunc := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(context.Background())
 	testCtx := TestContext{
 		Ctx:      ctx,
-		CancelFn: cancelFunc,
+		CancelFn: cancel,
 	}
 
 	testCtx.ClientSet, testCtx.KubeConfig, testCtx.CloseFn = framework.StartTestServer(t, framework.TestServerSetup{
@@ -354,6 +379,10 @@ func InitTestAPIServer(t *testing.T, nsPrefix string, admission admission.Interf
 		testCtx.NS = framework.CreateNamespaceOrDie(testCtx.ClientSet, "default", t)
 	}
 
+	t.Cleanup(func() {
+		CleanupTest(t, &testCtx)
+	})
+
 	return &testCtx
 }
 
@@ -388,6 +417,9 @@ func InitTestSchedulerWithOptions(
 	resyncPeriod time.Duration,
 	opts ...scheduler.Option,
 ) *TestContext {
+	ctx, cancel := context.WithCancel(testCtx.Ctx)
+	testCtx.SchedulerCtx = ctx
+
 	// 1. Create scheduler
 	testCtx.InformerFactory = scheduler.NewInformerFactory(testCtx.ClientSet, resyncPeriod)
 	if testCtx.KubeConfig != nil {
@@ -406,7 +438,7 @@ func InitTestSchedulerWithOptions(
 		testCtx.InformerFactory,
 		testCtx.DynInformerFactory,
 		profile.NewRecorderFactory(eventBroadcaster),
-		testCtx.Ctx.Done(),
+		ctx.Done(),
 		opts...,
 	)
 
@@ -414,13 +446,19 @@ func InitTestSchedulerWithOptions(
 		t.Fatalf("Couldn't create scheduler: %v", err)
 	}
 
-	eventBroadcaster.StartRecordingToSink(testCtx.Ctx.Done())
+	eventBroadcaster.StartRecordingToSink(ctx.Done())
 
 	oldCloseFn := testCtx.CloseFn
 	testCtx.CloseFn = func() {
 		oldCloseFn()
 		eventBroadcaster.Shutdown()
 	}
+
+	testCtx.SchedulerCloseFn = func() {
+		cancel()
+		eventBroadcaster.Shutdown()
+	}
+
 	return testCtx
 }
 
@@ -488,8 +526,8 @@ func InitDisruptionController(t *testing.T, testCtx *TestContext) *disruption.Di
 // configuration.
 func InitTestSchedulerWithNS(t *testing.T, nsPrefix string, opts ...scheduler.Option) *TestContext {
 	testCtx := InitTestSchedulerWithOptions(t, InitTestAPIServer(t, nsPrefix, nil), 0, opts...)
-	SyncInformerFactory(testCtx)
-	go testCtx.Scheduler.Run(testCtx.Ctx)
+	SyncSchedulerInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.SchedulerCtx)
 	return testCtx
 }
 
@@ -512,8 +550,8 @@ func InitTestDisablePreemption(t *testing.T, nsPrefix string) *TestContext {
 		t, InitTestAPIServer(t, nsPrefix, nil),
 		0,
 		scheduler.WithProfiles(cfg.Profiles...))
-	SyncInformerFactory(testCtx)
-	go testCtx.Scheduler.Run(testCtx.Ctx)
+	SyncSchedulerInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.SchedulerCtx)
 	return testCtx
 }