Merge pull request #46672 from smarterclayton/initializer_with_config

Automatic merge from submit-queue (batch tested with PRs 46967, 46992, 43338, 46717, 46672)

Select initializers from the dynamic configuration

Continues #36721

kubernetes/features#209

test/e2e/extension/BUILD

@@ -13,6 +13,9 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api/v1:go_default_library",
+        "//pkg/apis/admissionregistration/v1alpha1:go_default_library",
+        "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/retry:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",

test/e2e/extension/initializers.go

@@ -23,10 +23,14 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/apis/admissionregistration/v1alpha1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	clientretry "k8s.io/kubernetes/pkg/client/retry"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -65,10 +69,19 @@ var _ = framework.KubeDescribe("Initializers", func() {
 
 		// verify that we can update an initializing pod
 		pod, err := c.Core().Pods(ns).Get(podName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
 		pod.Annotations = map[string]string{"update-1": "test"}
 		pod, err = c.Core().Pods(ns).Update(pod)
 		Expect(err).NotTo(HaveOccurred())
 
+		// verify the list call filters out uninitialized pods
+		pods, err := c.Core().Pods(ns).List(metav1.ListOptions{IncludeUninitialized: true})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pods.Items).To(HaveLen(1))
+		pods, err = c.Core().Pods(ns).List(metav1.ListOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pods.Items).To(HaveLen(0))
+
 		// clear initializers
 		pod.Initializers = nil
 		pod, err = c.Core().Pods(ns).Update(pod)
@@ -93,17 +106,120 @@ var _ = framework.KubeDescribe("Initializers", func() {
 		}
 	})
 
+	It("should dynamically register and apply initializers to pods [Serial]", func() {
+		ns := f.Namespace.Name
+		c := f.ClientSet
+
+		podName := "uninitialized-pod"
+		framework.Logf("Creating pod %s", podName)
+
+		// create and register an initializer
+		initializerName := "pod.test.e2e.kubernetes.io"
+		initializerConfigName := "e2e-test-initializer"
+		_, err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&v1alpha1.InitializerConfiguration{
+			ObjectMeta: metav1.ObjectMeta{Name: initializerConfigName},
+			Initializers: []v1alpha1.Initializer{
+				{
+					Name: initializerName,
+					Rules: []v1alpha1.Rule{
+						{APIGroups: []string{""}, APIVersions: []string{"*"}, Resources: []string{"pods"}},
+					},
+				},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		// we must remove the initializer when the test is complete and ensure no pods are pending for that initializer
+		defer func() {
+			if err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Delete(initializerConfigName, nil); err != nil && !errors.IsNotFound(err) {
+				framework.Logf("got error on deleting %s", initializerConfigName)
+			}
+			// poller configuration is 1 second, wait at least that long
+			time.Sleep(3 * time.Second)
+			// clear our initializer from anyone who got it
+			removeInitializersFromAllPods(c, initializerName)
+		}()
+
+		// poller configuration is 1 second, wait at least that long
+		time.Sleep(3 * time.Second)
+
+		// run create that blocks
+		ch := make(chan struct{})
+		go func() {
+			defer close(ch)
+			_, err := c.Core().Pods(ns).Create(newPod(podName))
+			Expect(err).NotTo(HaveOccurred())
+		}()
+
+		// wait until the pod shows up uninitialized
+		By("Waiting until the pod is visible to a client")
+		var pod *v1.Pod
+		err = wait.PollImmediate(2*time.Second, 15*time.Second, func() (bool, error) {
+			pod, err = c.Core().Pods(ns).Get(podName, metav1.GetOptions{IncludeUninitialized: true})
+			if errors.IsNotFound(err) {
+				return false, nil
+			}
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pod.Initializers).NotTo(BeNil())
+		Expect(pod.Initializers.Pending).To(HaveLen(1))
+		Expect(pod.Initializers.Pending[0].Name).To(Equal(initializerName))
+
+		// pretend we are an initializer
+		By("Completing initialization")
+		pod.Initializers = nil
+		pod, err = c.Core().Pods(ns).Update(pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		// ensure create call returns
+		<-ch
+
+		// pod should now start running
+		err = framework.WaitForPodRunningInNamespace(c, pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		// bypass initialization by explicitly passing an empty pending list
+		By("Setting an empty initializer as an admin to bypass initialization")
+		podName = "preinitialized-pod"
+		pod = newUninitializedPod(podName)
+		pod.Initializers.Pending = nil
+		pod, err = c.Core().Pods(ns).Create(pod)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pod.Initializers).To(BeNil())
+
+		// bypass initialization for mirror pods
+		By("Creating a mirror pod that bypasses initialization")
+		podName = "mirror-pod"
+		pod = newPod(podName)
+		pod.Annotations = map[string]string{
+			v1.MirrorPodAnnotationKey: "true",
+		}
+		pod.Spec.NodeName = "node-does-not-yet-exist"
+		pod, err = c.Core().Pods(ns).Create(pod)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pod.Initializers).To(BeNil())
+		Expect(pod.Annotations[v1.MirrorPodAnnotationKey]).To(Equal("true"))
+	})
 })
 
 func newUninitializedPod(podName string) *v1.Pod {
+	pod := newPod(podName)
+	pod.Initializers = &metav1.Initializers{
+		Pending: []metav1.Initializer{{Name: "Test"}},
+	}
+	return pod
+}
+
+func newPod(podName string) *v1.Pod {
 	containerName := fmt.Sprintf("%s-container", podName)
 	port := 8080
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: podName,
-			Initializers: &metav1.Initializers{
-				Pending: []metav1.Initializer{{Name: "Test"}},
-			},
 		},
 		Spec: v1.PodSpec{
 			Containers: []v1.Container{
@@ -119,3 +235,48 @@ func newUninitializedPod(podName string) *v1.Pod {
 	}
 	return pod
 }
+
+// removeInitializersFromAllPods walks all pods and ensures they don't have the provided initializer,
+// to guarantee completing the test doesn't block the entire cluster.
+func removeInitializersFromAllPods(c clientset.Interface, initializerName string) {
+	pods, err := c.Core().Pods("").List(metav1.ListOptions{IncludeUninitialized: true})
+	if err != nil {
+		return
+	}
+	for _, p := range pods.Items {
+		if p.Initializers == nil {
+			continue
+		}
+		err := clientretry.RetryOnConflict(clientretry.DefaultRetry, func() error {
+			pod, err := c.Core().Pods(p.Namespace).Get(p.Name, metav1.GetOptions{IncludeUninitialized: true})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					return nil
+				}
+				return err
+			}
+			if pod.Initializers == nil {
+				return nil
+			}
+			var updated []metav1.Initializer
+			for _, pending := range pod.Initializers.Pending {
+				if pending.Name != initializerName {
+					updated = append(updated, pending)
+				}
+			}
+			if len(updated) == len(pod.Initializers.Pending) {
+				return nil
+			}
+			pod.Initializers.Pending = updated
+			if len(updated) == 0 {
+				pod.Initializers = nil
+			}
+			framework.Logf("Found initializer on pod %s in ns %s", pod.Name, pod.Namespace)
+			_, err = c.Core().Pods(p.Namespace).Update(pod)
+			return err
+		})
+		if err != nil {
+			framework.Logf("Unable to remove initializer from pod %s in ns %s: %v", p.Name, p.Namespace, err)
+		}
+	}
+}