12f96e2e35
Automatic merge from submit-queue (batch tested with PRs 51583, 51283, 51374, 51690, 51716) Unify initializer name validation Unify the validation rules on initializer names. Fix https://github.com/kubernetes/kubernetes/issues/51843. ```release-note Action required: validation rule on metadata.initializers.pending[x].name is tightened. The initializer name needs to contain at least three segments separated by dots. If you create objects with pending initializers, (i.e., not relying on apiserver adding pending initializers according to initializerconfiguration), you need to update the initializer name in existing objects and in configuration files to comply to the new validation rule. ```
418 lines
14 KiB
Go
418 lines
14 KiB
Go
/*
|
|
Copyright 2017 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apimachinery
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
|
|
"k8s.io/api/admissionregistration/v1alpha1"
|
|
"k8s.io/api/core/v1"
|
|
"k8s.io/api/extensions/v1beta1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
clientretry "k8s.io/client-go/util/retry"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
imageutils "k8s.io/kubernetes/test/utils/image"
|
|
)
|
|
|
|
var _ = SIGDescribe("Initializers [Feature:Initializers]", func() {
|
|
f := framework.NewDefaultFramework("initializers")
|
|
|
|
// TODO: Add failure traps once we have JustAfterEach
|
|
// See https://github.com/onsi/ginkgo/issues/303
|
|
|
|
It("should be invisible to controllers by default", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
ch := make(chan struct{})
|
|
go func() {
|
|
_, err := c.Core().Pods(ns).Create(newUninitializedPod(podName))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
close(ch)
|
|
}()
|
|
|
|
// wait to ensure the scheduler does not act on an uninitialized pod
|
|
err := wait.PollImmediate(2*time.Second, 15*time.Second, func() (bool, error) {
|
|
p, err := c.Core().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
return len(p.Spec.NodeName) > 0, nil
|
|
})
|
|
Expect(err).To(Equal(wait.ErrWaitTimeout))
|
|
|
|
// verify that we can update an initializing pod
|
|
pod, err := c.Core().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
pod.Annotations = map[string]string{"update-1": "test"}
|
|
pod, err = c.Core().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// verify the list call filters out uninitialized pods
|
|
pods, err := c.Core().Pods(ns).List(metav1.ListOptions{IncludeUninitialized: true})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pods.Items).To(HaveLen(1))
|
|
pods, err = c.Core().Pods(ns).List(metav1.ListOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pods.Items).To(HaveLen(0))
|
|
|
|
// clear initializers
|
|
pod.Initializers = nil
|
|
pod, err = c.Core().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// pod should now start running
|
|
err = framework.WaitForPodRunningInNamespace(c, pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// ensure create call returns
|
|
<-ch
|
|
|
|
// verify that we cannot start the pod initializing again
|
|
pod, err = c.Core().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
pod.Initializers = &metav1.Initializers{
|
|
Pending: []metav1.Initializer{{Name: "Other"}},
|
|
}
|
|
_, err = c.Core().Pods(ns).Update(pod)
|
|
if !errors.IsInvalid(err) || !strings.Contains(err.Error(), "immutable") {
|
|
Fail(fmt.Sprintf("expected invalid error: %v", err))
|
|
}
|
|
})
|
|
|
|
It("should dynamically register and apply initializers to pods [Serial]", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// create and register an initializer
|
|
initializerName := "pod.test.e2e.kubernetes.io"
|
|
initializerConfigName := "e2e-test-initializer"
|
|
_, err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&v1alpha1.InitializerConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{Name: initializerConfigName},
|
|
Initializers: []v1alpha1.Initializer{
|
|
{
|
|
Name: initializerName,
|
|
Rules: []v1alpha1.Rule{
|
|
{APIGroups: []string{""}, APIVersions: []string{"*"}, Resources: []string{"pods"}},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if errors.IsNotFound(err) {
|
|
framework.Skipf("dynamic configuration of initializers requires the alpha admissionregistration.k8s.io group to be enabled")
|
|
}
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// we must remove the initializer when the test is complete and ensure no pods are pending for that initializer
|
|
defer cleanupInitializer(c, initializerConfigName, initializerName)
|
|
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
|
|
// run create that blocks
|
|
ch := make(chan struct{})
|
|
go func() {
|
|
defer close(ch)
|
|
_, err := c.Core().Pods(ns).Create(newInitPod(podName))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
}()
|
|
|
|
// wait until the pod shows up uninitialized
|
|
By("Waiting until the pod is visible to a client")
|
|
var pod *v1.Pod
|
|
err = wait.PollImmediate(2*time.Second, 15*time.Second, func() (bool, error) {
|
|
pod, err = c.Core().Pods(ns).Get(podName, metav1.GetOptions{IncludeUninitialized: true})
|
|
if errors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).NotTo(BeNil())
|
|
Expect(pod.Initializers.Pending).To(HaveLen(1))
|
|
Expect(pod.Initializers.Pending[0].Name).To(Equal(initializerName))
|
|
|
|
// pretend we are an initializer
|
|
By("Completing initialization")
|
|
pod.Initializers = nil
|
|
pod, err = c.Core().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// ensure create call returns
|
|
<-ch
|
|
|
|
// pod should now start running
|
|
err = framework.WaitForPodRunningInNamespace(c, pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// bypass initialization by explicitly passing an empty pending list
|
|
By("Setting an empty initializer as an admin to bypass initialization")
|
|
podName = "preinitialized-pod"
|
|
pod = newUninitializedPod(podName)
|
|
pod.Initializers.Pending = nil
|
|
pod, err = c.Core().Pods(ns).Create(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).To(BeNil())
|
|
|
|
// bypass initialization for mirror pods
|
|
By("Creating a mirror pod that bypasses initialization")
|
|
podName = "mirror-pod"
|
|
pod = newInitPod(podName)
|
|
pod.Annotations = map[string]string{
|
|
v1.MirrorPodAnnotationKey: "true",
|
|
}
|
|
pod.Spec.NodeName = "node-does-not-yet-exist"
|
|
pod, err = c.Core().Pods(ns).Create(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).To(BeNil())
|
|
Expect(pod.Annotations[v1.MirrorPodAnnotationKey]).To(Equal("true"))
|
|
})
|
|
|
|
It("don't cause replicaset controller creating extra pods if the initializer is not handled [Serial]", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// create and register an initializer, without setting up a controller to handle it.
|
|
initializerName := "pod.test.e2e.kubernetes.io"
|
|
initializerConfigName := "e2e-test-initializer"
|
|
_, err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&v1alpha1.InitializerConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{Name: initializerConfigName},
|
|
Initializers: []v1alpha1.Initializer{
|
|
{
|
|
Name: initializerName,
|
|
Rules: []v1alpha1.Rule{
|
|
{APIGroups: []string{""}, APIVersions: []string{"*"}, Resources: []string{"pods"}},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if errors.IsNotFound(err) {
|
|
framework.Skipf("dynamic configuration of initializers requires the alpha admissionregistration.k8s.io group to be enabled")
|
|
}
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// we must remove the initializer when the test is complete and ensure no pods are pending for that initializer
|
|
defer cleanupInitializer(c, initializerConfigName, initializerName)
|
|
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
|
|
// create a replicaset
|
|
persistedRS, err := c.ExtensionsV1beta1().ReplicaSets(ns).Create(newReplicaset())
|
|
Expect(err).NotTo(HaveOccurred())
|
|
// wait for replicaset controller to confirm that it has handled the creation
|
|
err = waitForRSObservedGeneration(c, persistedRS.Namespace, persistedRS.Name, persistedRS.Generation)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// update the replicaset spec to trigger a resync
|
|
patch := []byte(`{"spec":{"minReadySeconds":5}}`)
|
|
persistedRS, err = c.ExtensionsV1beta1().ReplicaSets(ns).Patch(persistedRS.Name, types.StrategicMergePatchType, patch)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// wait for replicaset controller to confirm that it has handle the spec update
|
|
err = waitForRSObservedGeneration(c, persistedRS.Namespace, persistedRS.Name, persistedRS.Generation)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// verify that the replicaset controller doesn't create extra pod
|
|
selector, err := metav1.LabelSelectorAsSelector(persistedRS.Spec.Selector)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
listOptions := metav1.ListOptions{
|
|
LabelSelector: selector.String(),
|
|
IncludeUninitialized: true,
|
|
}
|
|
pods, err := c.Core().Pods(ns).List(listOptions)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(len(pods.Items)).Should(Equal(1))
|
|
})
|
|
|
|
It("will be set to nil if a patch removes the last pending initializer", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "to-be-patch-initialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// TODO: lower the timeout so that the server responds faster.
|
|
_, err := c.CoreV1().Pods(ns).Create(newUninitializedPod(podName))
|
|
if err != nil && !errors.IsTimeout(err) {
|
|
framework.Failf("expect err to be timeout error, got %v", err)
|
|
}
|
|
uninitializedPod, err := c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(uninitializedPod.Initializers).NotTo(BeNil())
|
|
Expect(len(uninitializedPod.Initializers.Pending)).Should(Equal(1))
|
|
|
|
patch := fmt.Sprintf(`{"metadata":{"initializers":{"pending":[{"$patch":"delete","name":"%s"}]}}}`, uninitializedPod.Initializers.Pending[0].Name)
|
|
patchedPod, err := c.CoreV1().Pods(ns).Patch(uninitializedPod.Name, types.StrategicMergePatchType, []byte(patch))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(patchedPod.Initializers).To(BeNil())
|
|
})
|
|
})
|
|
|
|
func newUninitializedPod(podName string) *v1.Pod {
|
|
pod := newInitPod(podName)
|
|
pod.Initializers = &metav1.Initializers{
|
|
Pending: []metav1.Initializer{{Name: "test.k8s.io"}},
|
|
}
|
|
return pod
|
|
}
|
|
|
|
func newReplicaset() *v1beta1.ReplicaSet {
|
|
name := "initializer-test-replicaset"
|
|
replicas := int32(1)
|
|
labels := map[string]string{"initializer-test": "single-replicaset"}
|
|
return &v1beta1.ReplicaSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: v1beta1.ReplicaSetSpec{
|
|
Replicas: &replicas,
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: labels,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
TerminationGracePeriodSeconds: &zero,
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: name + "-container",
|
|
Image: "gcr.io/google_containers/porter:4524579c0eb935c056c8e75563b4e1eda31587e0",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func newInitPod(podName string) *v1.Pod {
|
|
containerName := fmt.Sprintf("%s-container", podName)
|
|
port := 8080
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: containerName,
|
|
Image: imageutils.GetE2EImage(imageutils.Porter),
|
|
Env: []v1.EnvVar{{Name: fmt.Sprintf("SERVE_PORT_%d", port), Value: "foo"}},
|
|
Ports: []v1.ContainerPort{{ContainerPort: int32(port)}},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
},
|
|
}
|
|
return pod
|
|
}
|
|
|
|
// removeInitializersFromAllPods walks all pods and ensures they don't have the provided initializer,
|
|
// to guarantee completing the test doesn't block the entire cluster.
|
|
func removeInitializersFromAllPods(c clientset.Interface, initializerName string) {
|
|
pods, err := c.Core().Pods("").List(metav1.ListOptions{IncludeUninitialized: true})
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, p := range pods.Items {
|
|
if p.Initializers == nil {
|
|
continue
|
|
}
|
|
err := clientretry.RetryOnConflict(clientretry.DefaultRetry, func() error {
|
|
pod, err := c.Core().Pods(p.Namespace).Get(p.Name, metav1.GetOptions{IncludeUninitialized: true})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if pod.Initializers == nil {
|
|
return nil
|
|
}
|
|
var updated []metav1.Initializer
|
|
for _, pending := range pod.Initializers.Pending {
|
|
if pending.Name != initializerName {
|
|
updated = append(updated, pending)
|
|
}
|
|
}
|
|
if len(updated) == len(pod.Initializers.Pending) {
|
|
return nil
|
|
}
|
|
pod.Initializers.Pending = updated
|
|
if len(updated) == 0 {
|
|
pod.Initializers = nil
|
|
}
|
|
framework.Logf("Found initializer on pod %s in ns %s", pod.Name, pod.Namespace)
|
|
_, err = c.Core().Pods(p.Namespace).Update(pod)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
framework.Logf("Unable to remove initializer from pod %s in ns %s: %v", p.Name, p.Namespace, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// remove the initializerConfig, and remove the initializer from all pods
|
|
func cleanupInitializer(c clientset.Interface, initializerConfigName, initializerName string) {
|
|
if err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Delete(initializerConfigName, nil); err != nil && !errors.IsNotFound(err) {
|
|
framework.Logf("got error on deleting %s", initializerConfigName)
|
|
}
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
// clear our initializer from anyone who got it
|
|
removeInitializersFromAllPods(c, initializerName)
|
|
}
|
|
|
|
// waits till the RS status.observedGeneration matches metadata.generation.
|
|
func waitForRSObservedGeneration(c clientset.Interface, ns, name string, generation int64) error {
|
|
return wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
|
|
rs, err := c.Extensions().ReplicaSets(ns).Get(name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if generation > rs.Status.ObservedGeneration {
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
})
|
|
}
|