1b55f57391
Implements history utilities for ControllerRevision in the controller/history package StatefulSetStatus now has additional fields for consistency with DaemonSet and Deployment StatefulSetStatus.Replicas now represents the current number of createdPods and StatefulSetStatus.ReadyReplicas is the current number of ready Pods
2184 lines
68 KiB
Go
2184 lines
68 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package statefulset
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"math/rand"
|
|
"reflect"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/client-go/tools/cache"
|
|
"k8s.io/kubernetes/pkg/api"
|
|
"k8s.io/kubernetes/pkg/api/v1"
|
|
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
|
apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
|
|
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
|
|
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset/fake"
|
|
informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
|
|
appsinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions/apps/v1beta1"
|
|
coreinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions/core/v1"
|
|
appslisters "k8s.io/kubernetes/pkg/client/listers/apps/v1beta1"
|
|
corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1"
|
|
"k8s.io/kubernetes/pkg/controller"
|
|
"k8s.io/kubernetes/pkg/controller/history"
|
|
"k8s.io/metrics/pkg/client/clientset_generated/clientset/scheme"
|
|
)
|
|
|
|
type invariantFunc func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
|
|
func setupController(client clientset.Interface) (*fakeStatefulPodControl, *fakeStatefulSetStatusUpdater, StatefulSetControlInterface, chan struct{}) {
|
|
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
|
|
spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1beta1().StatefulSets())
|
|
ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1beta1().StatefulSets())
|
|
ssc := NewDefaultStatefulSetControl(spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1beta1().ControllerRevisions()))
|
|
|
|
stop := make(chan struct{})
|
|
informerFactory.Start(stop)
|
|
cache.WaitForCacheSync(
|
|
stop,
|
|
informerFactory.Apps().V1beta1().StatefulSets().Informer().HasSynced,
|
|
informerFactory.Core().V1().Pods().Informer().HasSynced,
|
|
informerFactory.Apps().V1beta1().ControllerRevisions().Informer().HasSynced,
|
|
)
|
|
return spc, ssu, ssc, stop
|
|
}
|
|
|
|
func burst(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.PodManagementPolicy = apps.ParallelPodManagement
|
|
return set
|
|
}
|
|
|
|
func TestStatefulSetControl(t *testing.T) {
|
|
simpleSetFn := func() *apps.StatefulSet { return newStatefulSet(3) }
|
|
largeSetFn := func() *apps.StatefulSet { return newStatefulSet(5) }
|
|
|
|
testCases := []struct {
|
|
fn func(*testing.T, *apps.StatefulSet, invariantFunc)
|
|
obj func() *apps.StatefulSet
|
|
}{
|
|
{CreatesPods, simpleSetFn},
|
|
{ScalesUp, simpleSetFn},
|
|
{ScalesDown, simpleSetFn},
|
|
{ReplacesPods, largeSetFn},
|
|
{RecreatesFailedPod, simpleSetFn},
|
|
{SetsInitAnnotation, simpleSetFn},
|
|
{CreatePodFailure, simpleSetFn},
|
|
{UpdatePodFailure, simpleSetFn},
|
|
{UpdateSetStatusFailure, simpleSetFn},
|
|
{PodRecreateDeleteFailure, simpleSetFn},
|
|
}
|
|
|
|
for _, testCase := range testCases {
|
|
fnName := runtime.FuncForPC(reflect.ValueOf(testCase.fn).Pointer()).Name()
|
|
if i := strings.LastIndex(fnName, "."); i != -1 {
|
|
fnName = fnName[i+1:]
|
|
}
|
|
t.Run(
|
|
fmt.Sprintf("%s/Monotonic", fnName),
|
|
func(t *testing.T) {
|
|
testCase.fn(t, testCase.obj(), assertMonotonicInvariants)
|
|
},
|
|
)
|
|
t.Run(
|
|
fmt.Sprintf("%s/Burst", fnName),
|
|
func(t *testing.T) {
|
|
set := burst(testCase.obj())
|
|
testCase.fn(t, set, assertBurstInvariants)
|
|
},
|
|
)
|
|
}
|
|
}
|
|
|
|
func CreatesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 3 {
|
|
t.Error("Failed to scale statefulset to 3 replicas")
|
|
}
|
|
}
|
|
|
|
func ScalesUp(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
*set.Spec.Replicas = 4
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to scale StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 4 {
|
|
t.Error("Failed to scale statefulset to 4 replicas")
|
|
}
|
|
}
|
|
|
|
func ScalesDown(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
*set.Spec.Replicas = 0
|
|
if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to scale StatefulSet : %s", err)
|
|
}
|
|
if set.Status.Replicas != 0 {
|
|
t.Error("Failed to scale statefulset to 0 replicas")
|
|
}
|
|
}
|
|
|
|
func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 5 {
|
|
t.Error("Failed to scale statefulset to 5 replicas")
|
|
}
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
spc.podsIndexer.Delete(pods[0])
|
|
spc.podsIndexer.Delete(pods[2])
|
|
spc.podsIndexer.Delete(pods[4])
|
|
for i := 0; i < 5; i += 2 {
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Failed to update StatefulSet : %s", err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if pods, err = spc.setPodRunning(set, i); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Failed to update StatefulSet : %s", err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if pods, err = spc.setPodReady(set, i); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Failed to update StatefulSet : %s", err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if e, a := int32(5), set.Status.Replicas; e != a {
|
|
t.Errorf("Expected to scale to %d, got %d", e, a)
|
|
}
|
|
}
|
|
|
|
func RecreatesFailedPod(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset()
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods[0].Status.Phase = v1.PodFailed
|
|
spc.podsIndexer.Update(pods[0])
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if isCreated(pods[0]) {
|
|
t.Error("StatefulSet did not recreate failed Pod")
|
|
}
|
|
}
|
|
|
|
func SetsInitAnnotation(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err = invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if pods, err = spc.setPodRunning(set, 0); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if pods, err = spc.setPodReady(set, 0); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if pods, err = spc.setPodInitStatus(set, 0, false); err != nil {
|
|
t.Error(err)
|
|
}
|
|
replicas := int(set.Status.Replicas)
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if replicas != int(set.Status.Replicas) {
|
|
t.Errorf("StatefulSetControl does not block on %s=false", apps.StatefulSetInitAnnotation)
|
|
}
|
|
if pods, err = spc.setPodInitStatus(set, 0, true); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if int(set.Status.Replicas) != 3 {
|
|
t.Errorf("StatefulSetControl does not unblock on %s=true", apps.StatefulSetInitAnnotation)
|
|
}
|
|
}
|
|
|
|
func CreatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
spc.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); !apierrors.IsInternalError(err) {
|
|
t.Errorf("StatefulSetControl did not return InternalError found %s", err)
|
|
}
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 3 {
|
|
t.Error("Failed to scale StatefulSet to 3 replicas")
|
|
}
|
|
}
|
|
|
|
func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
spc.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0)
|
|
|
|
// have to have 1 successful loop first
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Fatalf("Unexpected error: %v", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 3 {
|
|
t.Error("Failed to scale StatefulSet to 3 replicas")
|
|
}
|
|
|
|
// now mutate a pod's identity
|
|
pods, err := spc.podsLister.List(labels.Everything())
|
|
if err != nil {
|
|
t.Fatalf("Error listing pods: %v", err)
|
|
}
|
|
if len(pods) != 3 {
|
|
t.Fatalf("Expected 3 pods, got %d", len(pods))
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
pods[0].Name = "goo-0"
|
|
spc.podsIndexer.Update(pods[0])
|
|
|
|
// now it should fail
|
|
if err := ssc.UpdateStatefulSet(set, pods); !apierrors.IsInternalError(err) {
|
|
t.Errorf("StatefulSetControl did not return InternalError found %s", err)
|
|
}
|
|
}
|
|
|
|
func UpdateSetStatusFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, ssu, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); !apierrors.IsInternalError(err) {
|
|
t.Errorf("StatefulSetControl did not return InternalError found %s", err)
|
|
}
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 3 {
|
|
t.Error("Failed to scale StatefulSet to 3 replicas")
|
|
}
|
|
}
|
|
|
|
func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) {
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods[0].Status.Phase = v1.PodFailed
|
|
spc.podsIndexer.Update(pods[0])
|
|
spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0)
|
|
if err := ssc.UpdateStatefulSet(set, pods); !apierrors.IsInternalError(err) {
|
|
t.Errorf("StatefulSet failed to %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
t.Errorf("Error updating StatefulSet %s", err)
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
t.Error(err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
if isCreated(pods[0]) {
|
|
t.Error("StatefulSet did not recreate failed Pod")
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlScaleDownDeleteError(t *testing.T) {
|
|
invariants := assertMonotonicInvariants
|
|
set := newStatefulSet(3)
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
|
}
|
|
var err error
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
*set.Spec.Replicas = 0
|
|
spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2)
|
|
if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); !apierrors.IsInternalError(err) {
|
|
t.Errorf("StatefulSetControl failed to throw error on delete %s", err)
|
|
}
|
|
if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil {
|
|
t.Errorf("Failed to turn down StatefulSet %s", err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("Error getting updated StatefulSet: %v", err)
|
|
}
|
|
if set.Status.Replicas != 0 {
|
|
t.Error("Failed to scale statefulset to 0 replicas")
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControl_getSetRevisions(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
existing []*apps.ControllerRevision
|
|
set *apps.StatefulSet
|
|
expectedCount int
|
|
expectedCurrent *apps.ControllerRevision
|
|
expectedUpdate *apps.ControllerRevision
|
|
err bool
|
|
}
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
client := fake.NewSimpleClientset()
|
|
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
|
|
spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1beta1().StatefulSets())
|
|
ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1beta1().StatefulSets())
|
|
ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1beta1().ControllerRevisions())}
|
|
|
|
stop := make(chan struct{})
|
|
defer close(stop)
|
|
informerFactory.Start(stop)
|
|
cache.WaitForCacheSync(
|
|
stop,
|
|
informerFactory.Apps().V1beta1().StatefulSets().Informer().HasSynced,
|
|
informerFactory.Core().V1().Pods().Informer().HasSynced,
|
|
informerFactory.Apps().V1beta1().ControllerRevisions().Informer().HasSynced,
|
|
)
|
|
for i := range test.existing {
|
|
ssc.controllerHistory.CreateControllerRevision(test.set, test.existing[i])
|
|
}
|
|
revisions, err := ssc.ListRevisions(test.set)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, update, err := ssc.getStatefulSetRevisions(test.set, revisions)
|
|
revisions, err = ssc.ListRevisions(test.set)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(revisions) != test.expectedCount {
|
|
t.Errorf("%s: want %d revisions got %d", test.name, test.expectedCount, len(revisions))
|
|
}
|
|
if test.err && err != nil {
|
|
t.Errorf("%s: expected error", test.name)
|
|
}
|
|
if !test.err && !history.EqualRevision(current, test.expectedCurrent) {
|
|
t.Errorf("%s: for current want %v got %v", test.name, test.expectedCurrent, current)
|
|
}
|
|
if !test.err && !history.EqualRevision(update, test.expectedUpdate) {
|
|
t.Errorf("%s: for current want %v got %v", test.name, test.expectedUpdate, update)
|
|
}
|
|
if !test.err && test.expectedCurrent != nil && current != nil && test.expectedCurrent.Revision != current.Revision {
|
|
t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedCurrent.Revision, current.Revision)
|
|
}
|
|
if !test.err && test.expectedUpdate != nil && update != nil && test.expectedUpdate.Revision != update.Revision {
|
|
t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedUpdate.Revision, update.Revision)
|
|
}
|
|
}
|
|
|
|
updateRevision := func(cr *apps.ControllerRevision, revision int64) *apps.ControllerRevision {
|
|
obj, err := scheme.Scheme.DeepCopy(cr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
clone := obj.(*apps.ControllerRevision)
|
|
clone.Revision = revision
|
|
return clone
|
|
}
|
|
|
|
set := newStatefulSet(3)
|
|
rev0 := newRevisionOrDie(set, 1)
|
|
set1 := copySet(set)
|
|
set1.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
set1.Status.CurrentRevision = rev0.Name
|
|
rev1 := newRevisionOrDie(set1, 2)
|
|
set2 := copySet(set1)
|
|
set2.Spec.Template.Labels["new"] = "label"
|
|
set2.Status.CurrentRevision = rev0.Name
|
|
rev2 := newRevisionOrDie(set2, 3)
|
|
tests := []testcase{
|
|
{
|
|
name: "creates initial revision",
|
|
existing: nil,
|
|
set: set,
|
|
expectedCount: 1,
|
|
expectedCurrent: rev0,
|
|
expectedUpdate: rev0,
|
|
err: false,
|
|
},
|
|
{
|
|
name: "creates revision on update",
|
|
existing: []*apps.ControllerRevision{rev0},
|
|
set: set1,
|
|
expectedCount: 2,
|
|
expectedCurrent: rev0,
|
|
expectedUpdate: rev1,
|
|
err: false,
|
|
},
|
|
{
|
|
name: "must not recreate a new revision of same set",
|
|
existing: []*apps.ControllerRevision{rev0, rev1},
|
|
set: set1,
|
|
expectedCount: 2,
|
|
expectedCurrent: rev0,
|
|
expectedUpdate: rev1,
|
|
err: false,
|
|
},
|
|
{
|
|
name: "must rollback to a previous revision",
|
|
existing: []*apps.ControllerRevision{rev0, rev1, rev2},
|
|
set: set1,
|
|
expectedCount: 3,
|
|
expectedCurrent: rev0,
|
|
expectedUpdate: updateRevision(rev1, 4),
|
|
err: false,
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlRollingUpdate(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
initial func() *apps.StatefulSet
|
|
update func(set *apps.StatefulSet) *apps.StatefulSet
|
|
validate func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
}
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
set := test.initial()
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set = test.update(set)
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validate(set, pods); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
}
|
|
|
|
tests := []testcase{
|
|
{
|
|
name: "monotonic image update",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale up",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale down",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(5)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale up",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale down",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(5))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlOnDeleteUpdate(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
initial func() *apps.StatefulSet
|
|
update func(set *apps.StatefulSet) *apps.StatefulSet
|
|
validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
validateRestart func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
}
|
|
|
|
originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
set := test.initial()
|
|
set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType}
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set = test.update(set)
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validateUpdate(set, pods); err != nil {
|
|
for i := range pods {
|
|
t.Log(pods[i].Name)
|
|
}
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
|
|
}
|
|
replicas := *set.Spec.Replicas
|
|
*set.Spec.Replicas = 0
|
|
if err := scaleDownStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
*set.Spec.Replicas = replicas
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validateRestart(set, pods); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
}
|
|
|
|
tests := []testcase{
|
|
{
|
|
name: "monotonic image update",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale up",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 3 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale down",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(5)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale up",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 3 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 3 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale down",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(5))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRestart: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlPartitionUpdate(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
partition int32
|
|
invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
initial func() *apps.StatefulSet
|
|
update func(set *apps.StatefulSet) *apps.StatefulSet
|
|
validate func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
}
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
set := test.initial()
|
|
set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{
|
|
Type: apps.PartitionStatefulSetStrategyType,
|
|
Partition: func() *apps.PartitionStatefulSetStrategy {
|
|
return &apps.PartitionStatefulSetStrategy{Ordinal: test.partition}
|
|
}(),
|
|
}
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set = test.update(set)
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validate(set, pods); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
}
|
|
|
|
originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image
|
|
|
|
tests := []testcase{
|
|
{
|
|
name: "monotonic image update",
|
|
invariants: assertMonotonicInvariants,
|
|
partition: 2,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 2 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale up",
|
|
partition: 2,
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 2 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update",
|
|
partition: 2,
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 2 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale up",
|
|
invariants: assertBurstInvariants,
|
|
partition: 2,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if i < 2 && pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
if i >= 2 && pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlLimitsHistory(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
initial func() *apps.StatefulSet
|
|
}
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
set := test.initial()
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
for i := 0; i < 10; i++ {
|
|
set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i)
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
err = ssc.UpdateStatefulSet(set, pods)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
revisions, err := ssc.ListRevisions(set)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 {
|
|
t.Fatalf("%s: %d greater than limit %d", test.name, len(revisions), *set.Spec.RevisionHistoryLimit)
|
|
}
|
|
}
|
|
}
|
|
|
|
tests := []testcase{
|
|
{
|
|
name: "monotonic update",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
},
|
|
{
|
|
name: "burst update",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
func TestStatefulSetControlRollback(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error
|
|
initial func() *apps.StatefulSet
|
|
update func(set *apps.StatefulSet) *apps.StatefulSet
|
|
validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
validateRollback func(set *apps.StatefulSet, pods []*v1.Pod) error
|
|
}
|
|
|
|
originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image
|
|
|
|
testFn := func(test *testcase, t *testing.T) {
|
|
set := test.initial()
|
|
client := fake.NewSimpleClientset(set)
|
|
spc, _, ssc, stop := setupController(client)
|
|
defer close(stop)
|
|
if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set = test.update(set)
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validateUpdate(set, pods); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
revisions, err := ssc.ListRevisions(set)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
history.SortControllerRevisions(revisions)
|
|
set, err = applyRevision(set, revisions[0])
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
if err := test.validateRollback(set, pods); err != nil {
|
|
t.Fatalf("%s: %s", test.name, err)
|
|
}
|
|
}
|
|
|
|
tests := []testcase{
|
|
{
|
|
name: "monotonic image update",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale up",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(3)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "monotonic image update and scale down",
|
|
invariants: assertMonotonicInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return newStatefulSet(5)
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale up",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(3))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 5
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
name: "burst image update and scale down",
|
|
invariants: assertBurstInvariants,
|
|
initial: func() *apps.StatefulSet {
|
|
return burst(newStatefulSet(5))
|
|
},
|
|
update: func(set *apps.StatefulSet) *apps.StatefulSet {
|
|
*set.Spec.Replicas = 3
|
|
set.Spec.Template.Spec.Containers[0].Image = "foo"
|
|
return set
|
|
},
|
|
validateUpdate: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != "foo" {
|
|
return fmt.Errorf("want pod %s image foo found %s", pods[i].Name, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
validateRollback: func(set *apps.StatefulSet, pods []*v1.Pod) error {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for i := range pods {
|
|
if pods[i].Spec.Containers[0].Image != originalImage {
|
|
return fmt.Errorf("want pod %s image %s found %s", pods[i].Name, originalImage, pods[i].Spec.Containers[0].Image)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
}
|
|
for i := range tests {
|
|
testFn(&tests[i], t)
|
|
}
|
|
}
|
|
|
|
type requestTracker struct {
|
|
requests int
|
|
err error
|
|
after int
|
|
}
|
|
|
|
func (rt *requestTracker) errorReady() bool {
|
|
return rt.err != nil && rt.requests >= rt.after
|
|
}
|
|
|
|
func (rt *requestTracker) inc() {
|
|
rt.requests++
|
|
}
|
|
|
|
func (rt *requestTracker) reset() {
|
|
rt.err = nil
|
|
rt.after = 0
|
|
}
|
|
|
|
type fakeStatefulPodControl struct {
|
|
podsLister corelisters.PodLister
|
|
claimsLister corelisters.PersistentVolumeClaimLister
|
|
setsLister appslisters.StatefulSetLister
|
|
podsIndexer cache.Indexer
|
|
claimsIndexer cache.Indexer
|
|
setsIndexer cache.Indexer
|
|
createPodTracker requestTracker
|
|
updatePodTracker requestTracker
|
|
deletePodTracker requestTracker
|
|
}
|
|
|
|
func newFakeStatefulPodControl(podInformer coreinformers.PodInformer, setInformer appsinformers.StatefulSetInformer) *fakeStatefulPodControl {
|
|
claimsIndexer := cache.NewIndexer(controller.KeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
|
|
return &fakeStatefulPodControl{
|
|
podInformer.Lister(),
|
|
corelisters.NewPersistentVolumeClaimLister(claimsIndexer),
|
|
setInformer.Lister(),
|
|
podInformer.Informer().GetIndexer(),
|
|
claimsIndexer,
|
|
setInformer.Informer().GetIndexer(),
|
|
requestTracker{0, nil, 0},
|
|
requestTracker{0, nil, 0},
|
|
requestTracker{0, nil, 0}}
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) SetCreateStatefulPodError(err error, after int) {
|
|
spc.createPodTracker.err = err
|
|
spc.createPodTracker.after = after
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) SetUpdateStatefulPodError(err error, after int) {
|
|
spc.updatePodTracker.err = err
|
|
spc.updatePodTracker.after = after
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) SetDeleteStatefulPodError(err error, after int) {
|
|
spc.deletePodTracker.err = err
|
|
spc.deletePodTracker.after = after
|
|
}
|
|
|
|
func copyPod(pod *v1.Pod) *v1.Pod {
|
|
obj, err := api.Scheme.Copy(pod)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return obj.(*v1.Pod)
|
|
}
|
|
|
|
func copySet(set *apps.StatefulSet) *apps.StatefulSet {
|
|
obj, err := scheme.Scheme.Copy(set)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return obj.(*apps.StatefulSet)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) setPodPending(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if 0 > ordinal || ordinal >= len(pods) {
|
|
return nil, fmt.Errorf("ordinal %d out of range [0,%d)", ordinal, len(pods))
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
pod := copyPod(pods[ordinal])
|
|
pod.Status.Phase = v1.PodPending
|
|
fakeResourceVersion(pod)
|
|
spc.podsIndexer.Update(pod)
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) setPodRunning(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if 0 > ordinal || ordinal >= len(pods) {
|
|
return nil, fmt.Errorf("ordinal %d out of range [0,%d)", ordinal, len(pods))
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
pod := copyPod(pods[ordinal])
|
|
pod.Status.Phase = v1.PodRunning
|
|
fakeResourceVersion(pod)
|
|
spc.podsIndexer.Update(pod)
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) setPodReady(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if 0 > ordinal || ordinal >= len(pods) {
|
|
return nil, fmt.Errorf("ordinal %d out of range [0,%d)", ordinal, len(pods))
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
pod := copyPod(pods[ordinal])
|
|
condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue}
|
|
podutil.UpdatePodCondition(&pod.Status, &condition)
|
|
fakeResourceVersion(pod)
|
|
spc.podsIndexer.Update(pod)
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) setPodInitStatus(set *apps.StatefulSet, ordinal int, init bool) ([]*v1.Pod, error) {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if 0 > ordinal || ordinal >= len(pods) {
|
|
return nil, fmt.Errorf("ordinal %d out of range [0,%d)", ordinal, len(pods))
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
pod := copyPod(pods[ordinal])
|
|
if init {
|
|
pod.Annotations[apps.StatefulSetInitAnnotation] = "true"
|
|
} else {
|
|
pod.Annotations[apps.StatefulSetInitAnnotation] = "false"
|
|
}
|
|
fakeResourceVersion(pod)
|
|
spc.podsIndexer.Update(pod)
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) addTerminatingPod(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) {
|
|
pod := newStatefulSetPod(set, ordinal)
|
|
pod.Status.Phase = v1.PodRunning
|
|
deleted := metav1.NewTime(time.Now())
|
|
pod.DeletionTimestamp = &deleted
|
|
condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue}
|
|
fakeResourceVersion(pod)
|
|
podutil.UpdatePodCondition(&pod.Status, &condition)
|
|
spc.podsIndexer.Update(pod)
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) setPodTerminated(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) {
|
|
pod := newStatefulSetPod(set, ordinal)
|
|
deleted := metav1.NewTime(time.Now())
|
|
pod.DeletionTimestamp = &deleted
|
|
fakeResourceVersion(pod)
|
|
spc.podsIndexer.Update(pod)
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return spc.podsLister.Pods(set.Namespace).List(selector)
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) CreateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error {
|
|
defer spc.createPodTracker.inc()
|
|
if spc.createPodTracker.errorReady() {
|
|
defer spc.createPodTracker.reset()
|
|
return spc.createPodTracker.err
|
|
}
|
|
|
|
for _, claim := range getPersistentVolumeClaims(set, pod) {
|
|
spc.claimsIndexer.Update(&claim)
|
|
}
|
|
spc.podsIndexer.Update(pod)
|
|
return nil
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error {
|
|
defer spc.updatePodTracker.inc()
|
|
if spc.updatePodTracker.errorReady() {
|
|
defer spc.updatePodTracker.reset()
|
|
return spc.updatePodTracker.err
|
|
}
|
|
if !identityMatches(set, pod) {
|
|
updateIdentity(set, pod)
|
|
}
|
|
if !storageMatches(set, pod) {
|
|
updateStorage(set, pod)
|
|
for _, claim := range getPersistentVolumeClaims(set, pod) {
|
|
spc.claimsIndexer.Update(&claim)
|
|
}
|
|
}
|
|
spc.podsIndexer.Update(pod)
|
|
return nil
|
|
}
|
|
|
|
func (spc *fakeStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error {
|
|
defer spc.deletePodTracker.inc()
|
|
if spc.deletePodTracker.errorReady() {
|
|
defer spc.deletePodTracker.reset()
|
|
return spc.deletePodTracker.err
|
|
}
|
|
if key, err := controller.KeyFunc(pod); err != nil {
|
|
return err
|
|
} else if obj, found, err := spc.podsIndexer.GetByKey(key); err != nil {
|
|
return err
|
|
} else if found {
|
|
spc.podsIndexer.Delete(obj)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var _ StatefulPodControlInterface = &fakeStatefulPodControl{}
|
|
|
|
type fakeStatefulSetStatusUpdater struct {
|
|
setsLister appslisters.StatefulSetLister
|
|
setsIndexer cache.Indexer
|
|
updateStatusTracker requestTracker
|
|
}
|
|
|
|
func newFakeStatefulSetStatusUpdater(setInformer appsinformers.StatefulSetInformer) *fakeStatefulSetStatusUpdater {
|
|
return &fakeStatefulSetStatusUpdater{
|
|
setInformer.Lister(),
|
|
setInformer.Informer().GetIndexer(),
|
|
requestTracker{0, nil, 0},
|
|
}
|
|
}
|
|
|
|
func (ssu *fakeStatefulSetStatusUpdater) UpdateStatefulSetStatus(set *apps.StatefulSet, status *apps.StatefulSetStatus) error {
|
|
defer ssu.updateStatusTracker.inc()
|
|
if ssu.updateStatusTracker.errorReady() {
|
|
defer ssu.updateStatusTracker.reset()
|
|
return ssu.updateStatusTracker.err
|
|
}
|
|
set.Status = *status
|
|
ssu.setsIndexer.Update(set)
|
|
return nil
|
|
}
|
|
|
|
func (ssu *fakeStatefulSetStatusUpdater) SetUpdateStatefulSetStatusError(err error, after int) {
|
|
ssu.updateStatusTracker.err = err
|
|
ssu.updateStatusTracker.after = after
|
|
}
|
|
|
|
var _ StatefulSetStatusUpdaterInterface = &fakeStatefulSetStatusUpdater{}
|
|
|
|
func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for ord := 0; ord < len(pods); ord++ {
|
|
if ord > 0 && isRunningAndReady(pods[ord]) && !isRunningAndReady(pods[ord-1]) {
|
|
return fmt.Errorf("Successor %s is Running and Ready while %s is not", pods[ord].Name, pods[ord-1].Name)
|
|
}
|
|
|
|
if getOrdinal(pods[ord]) != ord {
|
|
return fmt.Errorf("pods %s deployed in the wrong order %d", pods[ord].Name, ord)
|
|
}
|
|
|
|
if !storageMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pods %s does not match the storage specification of StatefulSet %s ", pods[ord].Name, set.Name)
|
|
}
|
|
|
|
for _, claim := range getPersistentVolumeClaims(set, pods[ord]) {
|
|
claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if claim == nil {
|
|
return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name)
|
|
}
|
|
}
|
|
|
|
if !identityMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pods %s does not match the identity specification of StatefulSet %s ", pods[ord].Name, set.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for ord := 0; ord < len(pods); ord++ {
|
|
if !storageMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pods %s does not match the storage specification of StatefulSet %s ", pods[ord].Name, set.Name)
|
|
}
|
|
|
|
for _, claim := range getPersistentVolumeClaims(set, pods[ord]) {
|
|
claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if claim == nil {
|
|
return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name)
|
|
}
|
|
}
|
|
|
|
if !identityMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pods %s does not match the identity specification of StatefulSet %s ",
|
|
pods[ord].Name,
|
|
set.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
for ord := 0; ord < len(pods); ord++ {
|
|
|
|
if !storageMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pod %s does not match the storage specification of StatefulSet %s ", pods[ord].Name, set.Name)
|
|
}
|
|
|
|
for _, claim := range getPersistentVolumeClaims(set, pods[ord]) {
|
|
claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if claim == nil {
|
|
return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name)
|
|
}
|
|
}
|
|
|
|
if !identityMatches(set, pods[ord]) {
|
|
return fmt.Errorf("pod %s does not match the identity specification of StatefulSet %s ", pods[ord].Name, set.Name)
|
|
}
|
|
}
|
|
if set.Spec.UpdateStrategy.Type == apps.OnDeleteStatefulSetStrategyType {
|
|
return nil
|
|
}
|
|
if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType {
|
|
for i := 0; i < int(set.Status.CurrentReplicas) && i < len(pods); i++ {
|
|
if want, got := set.Status.CurrentRevision, getPodRevision(pods[i]); want != got {
|
|
return fmt.Errorf("pod %s want current revision %s got %s", pods[i].Name, want, got)
|
|
}
|
|
}
|
|
for i, j := len(pods)-1, 0; j < int(set.Status.UpdatedReplicas); i, j = i-1, j+1 {
|
|
if want, got := set.Status.UpdateRevision, getPodRevision(pods[i]); want != got {
|
|
return fmt.Errorf("pod %s want update revision %s got %s", pods[i].Name, want, got)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fakeResourceVersion(object interface{}) {
|
|
obj, isObj := object.(metav1.Object)
|
|
if !isObj {
|
|
return
|
|
}
|
|
if version := obj.GetResourceVersion(); version == "" {
|
|
obj.SetResourceVersion("1")
|
|
} else if intValue, err := strconv.ParseInt(version, 10, 32); err == nil {
|
|
obj.SetResourceVersion(strconv.FormatInt(intValue+1, 10))
|
|
}
|
|
}
|
|
|
|
func scaleUpStatefulSetControl(set *apps.StatefulSet,
|
|
ssc StatefulSetControlInterface,
|
|
spc *fakeStatefulPodControl,
|
|
invariants invariantFunc) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for set.Status.ReadyReplicas < *set.Spec.Replicas {
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
|
|
// ensure all pods are valid (have a phase)
|
|
initialized := false
|
|
for ord, pod := range pods {
|
|
if pod.Status.Phase == "" {
|
|
if pods, err = spc.setPodPending(set, ord); err != nil {
|
|
return err
|
|
}
|
|
break
|
|
}
|
|
}
|
|
if initialized {
|
|
continue
|
|
}
|
|
|
|
// select one of the pods and move it forward in status
|
|
if len(pods) > 0 {
|
|
ord := int(rand.Int63n(int64(len(pods))))
|
|
pod := pods[ord]
|
|
switch pod.Status.Phase {
|
|
case v1.PodPending:
|
|
if pods, err = spc.setPodRunning(set, ord); err != nil {
|
|
return err
|
|
}
|
|
case v1.PodRunning:
|
|
if pods, err = spc.setPodReady(set, ord); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
continue
|
|
}
|
|
}
|
|
|
|
// run the controller once and check invariants
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return invariants(set, spc)
|
|
}
|
|
|
|
func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, spc *fakeStatefulPodControl, invariants invariantFunc) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for set.Status.Replicas > *set.Spec.Replicas {
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
if ordinal := len(pods) - 1; ordinal >= 0 {
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if pods, err = spc.addTerminatingPod(set, ordinal); err != nil {
|
|
return err
|
|
}
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
|
|
if len(pods) > 0 {
|
|
spc.podsIndexer.Delete(pods[len(pods)-1])
|
|
}
|
|
}
|
|
if err := ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return invariants(set, spc)
|
|
}
|
|
|
|
func updateComplete(set *apps.StatefulSet, pods []*v1.Pod) bool {
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
if len(pods) != int(*set.Spec.Replicas) {
|
|
return false
|
|
}
|
|
if set.Status.ReadyReplicas != *set.Spec.Replicas {
|
|
return false
|
|
}
|
|
|
|
if set.Spec.UpdateStrategy.Type == apps.RollingUpdateStatefulSetStrategyType {
|
|
if set.Status.CurrentReplicas < *set.Spec.Replicas {
|
|
return false
|
|
}
|
|
for i := range pods {
|
|
if getPodRevision(pods[i]) != set.Status.CurrentRevision {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
if set.Spec.UpdateStrategy.Type == apps.PartitionStatefulSetStrategyType {
|
|
if set.Status.UpdatedReplicas < (*set.Spec.Replicas - set.Spec.UpdateStrategy.Partition.Ordinal) {
|
|
return false
|
|
}
|
|
for i := 0; i < int(set.Spec.UpdateStrategy.Partition.Ordinal); i++ {
|
|
if getPodRevision(pods[i]) != set.Status.CurrentRevision {
|
|
return false
|
|
}
|
|
}
|
|
for i := int(set.Spec.UpdateStrategy.Partition.Ordinal); i < int(*set.Spec.Replicas); i++ {
|
|
if getPodRevision(pods[i]) != set.Status.UpdateRevision {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func updateStatefulSetControl(set *apps.StatefulSet,
|
|
ssc StatefulSetControlInterface,
|
|
spc *fakeStatefulPodControl,
|
|
invariants invariantFunc) error {
|
|
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for !updateComplete(set, pods) {
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sort.Sort(ascendingOrdinal(pods))
|
|
initialized := false
|
|
for ord, pod := range pods {
|
|
if pod.Status.Phase == "" {
|
|
if pods, err = spc.setPodPending(set, ord); err != nil {
|
|
return err
|
|
}
|
|
break
|
|
}
|
|
}
|
|
if initialized {
|
|
continue
|
|
}
|
|
|
|
if len(pods) > 0 {
|
|
ord := int(rand.Int63n(int64(len(pods))))
|
|
pod := pods[ord]
|
|
switch pod.Status.Phase {
|
|
case v1.PodPending:
|
|
if pods, err = spc.setPodRunning(set, ord); err != nil {
|
|
return err
|
|
}
|
|
case v1.PodRunning:
|
|
if pods, err = spc.setPodReady(set, ord); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
continue
|
|
}
|
|
}
|
|
|
|
if err = ssc.UpdateStatefulSet(set, pods); err != nil {
|
|
return err
|
|
}
|
|
set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := invariants(set, spc); err != nil {
|
|
return err
|
|
}
|
|
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return invariants(set, spc)
|
|
}
|
|
|
|
func newRevisionOrDie(set *apps.StatefulSet, revision int64) *apps.ControllerRevision {
|
|
rev, err := newRevision(set, revision)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return rev
|
|
}
|