Merge pull request #99728 from mattcary/control

StatefulSet PVC auto-delete implementation
This commit is contained in:
Kubernetes Prow Robot 2021-11-18 03:37:02 -08:00 committed by GitHub
commit b8af116327
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 4956 additions and 1139 deletions

View File

@ -1318,6 +1318,20 @@
} }
] ]
}, },
"io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy": {
"description": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"properties": {
"whenDeleted": {
"description": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"type": "string"
},
"whenScaled": {
"description": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
"type": "string"
}
},
"type": "object"
},
"io.k8s.api.apps.v1.StatefulSetSpec": { "io.k8s.api.apps.v1.StatefulSetSpec": {
"description": "A StatefulSetSpec is the specification of a StatefulSet.", "description": "A StatefulSetSpec is the specification of a StatefulSet.",
"properties": { "properties": {
@ -1326,6 +1340,10 @@
"format": "int32", "format": "int32",
"type": "integer" "type": "integer"
}, },
"persistentVolumeClaimRetentionPolicy": {
"$ref": "#/definitions/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy",
"description": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional"
},
"podManagementPolicy": { "podManagementPolicy": {
"description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.", "description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.",
"enum": [ "enum": [

View File

@ -8822,6 +8822,20 @@
} }
] ]
}, },
"io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy": {
"description": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"type": "object",
"properties": {
"whenDeleted": {
"description": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"type": "string"
},
"whenScaled": {
"description": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
"type": "string"
}
}
},
"io.k8s.api.apps.v1.StatefulSetSpec": { "io.k8s.api.apps.v1.StatefulSetSpec": {
"description": "A StatefulSetSpec is the specification of a StatefulSet.", "description": "A StatefulSetSpec is the specification of a StatefulSet.",
"type": "object", "type": "object",
@ -8836,6 +8850,10 @@
"type": "integer", "type": "integer",
"format": "int32" "format": "int32"
}, },
"persistentVolumeClaimRetentionPolicy": {
"description": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional",
"$ref": "#/components/schemas/io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy"
},
"podManagementPolicy": { "podManagementPolicy": {
"description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.", "description": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.\n\nPossible enum values:\n - `\"OrderedReady\"` will create pods in strictly increasing order on scale up and strictly decreasing order on scale down, progressing only when the previous pod is ready or terminated. At most one pod will be changed at any time.\n - `\"Parallel\"` will create and delete pods as soon as the stateful set replica count is changed, and will not wait for pods to be ready or complete termination.",
"type": "string", "type": "string",

View File

@ -40,6 +40,17 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
if len(s.Spec.UpdateStrategy.Type) == 0 { if len(s.Spec.UpdateStrategy.Type) == 0 {
s.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType s.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType
} }
if s.Spec.PersistentVolumeClaimRetentionPolicy == nil {
policies := []apps.PersistentVolumeClaimRetentionPolicyType{
apps.RetainPersistentVolumeClaimRetentionPolicyType,
apps.DeletePersistentVolumeClaimRetentionPolicyType,
}
choice := int32(c.Rand.Int31())
s.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: policies[choice&1],
WhenScaled: policies[(choice>>1)&1],
}
}
if s.Spec.RevisionHistoryLimit == nil { if s.Spec.RevisionHistoryLimit == nil {
s.Spec.RevisionHistoryLimit = new(int32) s.Spec.RevisionHistoryLimit = new(int32)
*s.Spec.RevisionHistoryLimit = 10 *s.Spec.RevisionHistoryLimit = 10

View File

@ -97,6 +97,40 @@ type RollingUpdateStatefulSetStrategy struct {
Partition int32 Partition int32
} }
// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
// deleted or scaled down.
type PersistentVolumeClaimRetentionPolicyType string
const (
// RetainPersistentVolumeClaimRetentionPolicyType is the default
// PersistentVolumeClaimRetentionPolicy and specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will not be deleted.
RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
// DeletePersistentVolumeClaimRetentionPolicyType specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will be deleted in the scenario specified in
// StatefulSetPersistentVolumeClaimPolicy.
DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
)
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
WhenDeleted PersistentVolumeClaimRetentionPolicyType
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
WhenScaled PersistentVolumeClaimRetentionPolicyType
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
type StatefulSetSpec struct { type StatefulSetSpec struct {
// Replicas is the desired number of replicas of the given Template. // Replicas is the desired number of replicas of the given Template.
@ -164,6 +198,12 @@ type StatefulSetSpec struct {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
MinReadySeconds int32 MinReadySeconds int32
// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -20,6 +20,8 @@ import (
appsv1 "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
) )
func addDefaultingFuncs(scheme *runtime.Scheme) error { func addDefaultingFuncs(scheme *runtime.Scheme) error {
@ -117,6 +119,18 @@ func SetDefaults_StatefulSet(obj *appsv1.StatefulSet) {
*obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0 *obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1.RetainPersistentVolumeClaimRetentionPolicyType
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1.RetainPersistentVolumeClaimRetentionPolicyType
}
}
if obj.Spec.Replicas == nil { if obj.Spec.Replicas == nil {
obj.Spec.Replicas = new(int32) obj.Spec.Replicas = new(int32)
*obj.Spec.Replicas = 1 *obj.Spec.Replicas = 1

View File

@ -27,10 +27,13 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/legacyscheme"
_ "k8s.io/kubernetes/pkg/apis/apps/install" _ "k8s.io/kubernetes/pkg/apis/apps/install"
. "k8s.io/kubernetes/pkg/apis/apps/v1" . "k8s.io/kubernetes/pkg/apis/apps/v1"
_ "k8s.io/kubernetes/pkg/apis/core/install" _ "k8s.io/kubernetes/pkg/apis/core/install"
"k8s.io/kubernetes/pkg/features"
utilpointer "k8s.io/utils/pointer" utilpointer "k8s.io/utils/pointer"
) )
@ -193,10 +196,13 @@ func TestSetDefaultStatefulSet(t *testing.T) {
} }
tests := []struct { tests := []struct {
original *appsv1.StatefulSet name string
expected *appsv1.StatefulSet original *appsv1.StatefulSet
expected *appsv1.StatefulSet
enablePVCDeletionPolicy bool
}{ }{
{ // labels and default update strategy {
name: "labels and default update strategy",
original: &appsv1.StatefulSet{ original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{ Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate, Template: defaultTemplate,
@ -221,7 +227,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
}, },
}, },
}, },
{ // Alternate update strategy {
name: "Alternate update strategy",
original: &appsv1.StatefulSet{ original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{ Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate, Template: defaultTemplate,
@ -246,7 +253,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
}, },
}, },
}, },
{ // Parallel pod management policy. {
name: "Parallel pod management policy",
original: &appsv1.StatefulSet{ original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{ Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate, Template: defaultTemplate,
@ -272,7 +280,8 @@ func TestSetDefaultStatefulSet(t *testing.T) {
}, },
}, },
}, },
{ // UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set {
name: "UpdateStrategy.RollingUpdate.Partition is not lost when UpdateStrategy.Type is not set",
original: &appsv1.StatefulSet{ original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{ Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate, Template: defaultTemplate,
@ -302,20 +311,151 @@ func TestSetDefaultStatefulSet(t *testing.T) {
}, },
}, },
}, },
{
name: "PVC delete policy enabled, no policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy enabled, with scaledown policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy disabled, with set deletion policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: true,
},
{
name: "PVC delete policy disabled, with policy specified",
original: &appsv1.StatefulSet{
Spec: appsv1.StatefulSetSpec{
Template: defaultTemplate,
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
},
expected: &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Labels: defaultLabels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &defaultReplicas,
Template: defaultTemplate,
PodManagementPolicy: appsv1.OrderedReadyPodManagement,
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
Partition: &defaultPartition,
},
},
PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: utilpointer.Int32Ptr(10),
},
},
enablePVCDeletionPolicy: false,
},
} }
for i, test := range tests { for _, test := range tests {
original := test.original test := test
expected := test.expected t.Run(test.name, func(t *testing.T) {
obj2 := roundTrip(t, runtime.Object(original)) defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, test.enablePVCDeletionPolicy)()
got, ok := obj2.(*appsv1.StatefulSet)
if !ok { obj2 := roundTrip(t, runtime.Object(test.original))
t.Errorf("(%d) unexpected object: %v", i, got) got, ok := obj2.(*appsv1.StatefulSet)
t.FailNow() if !ok {
} t.Errorf("unexpected object: %v", got)
if !apiequality.Semantic.DeepEqual(got.Spec, expected.Spec) { t.FailNow()
t.Errorf("(%d) got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", i, got.Spec, expected.Spec) }
} if !apiequality.Semantic.DeepEqual(got.Spec, test.expected.Spec) {
t.Errorf("got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", got.Spec, test.expected.Spec)
}
})
} }
} }

View File

@ -261,6 +261,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil { }); err != nil {
return err return err
} }
if err := s.AddGeneratedConversionFunc((*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { if err := s.AddGeneratedConversionFunc((*v1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) return Convert_v1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope)
}); err != nil { }); err != nil {
@ -1154,6 +1164,28 @@ func Convert_apps_StatefulSetList_To_v1_StatefulSetList(in *apps.StatefulSetList
return autoConvert_apps_StatefulSetList_To_v1_StatefulSetList(in, out, s) return autoConvert_apps_StatefulSetList_To_v1_StatefulSetList(in, out, s)
} }
func autoConvert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_v1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = v1.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = v1.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_v1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { func autoConvert_v1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error {
if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil {
return err return err
@ -1170,6 +1202,7 @@ func autoConvert_v1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1.StatefulSetSp
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }
@ -1189,6 +1222,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1_StatefulSetSpec(in *apps.StatefulSet
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*v1.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }

View File

@ -21,6 +21,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
) )
func addDefaultingFuncs(scheme *runtime.Scheme) error { func addDefaultingFuncs(scheme *runtime.Scheme) error {
@ -46,6 +48,19 @@ func SetDefaults_StatefulSet(obj *appsv1beta1.StatefulSet) {
obj.Labels = labels obj.Labels = labels
} }
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType
}
}
if obj.Spec.Replicas == nil { if obj.Spec.Replicas == nil {
obj.Spec.Replicas = new(int32) obj.Spec.Replicas = new(int32)
*obj.Spec.Replicas = 1 *obj.Spec.Replicas = 1

View File

@ -212,6 +212,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil { }); err != nil {
return err return err
} }
if err := s.AddGeneratedConversionFunc((*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { if err := s.AddGeneratedConversionFunc((*v1beta1.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) return Convert_v1beta1_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta1.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope)
}); err != nil { }); err != nil {
@ -818,6 +828,28 @@ func Convert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in *apps.StatefulSe
return autoConvert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in, out, s) return autoConvert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in, out, s)
} }
func autoConvert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = v1beta1.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = v1beta1.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta1_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_v1beta1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { func autoConvert_v1beta1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta1.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error {
if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil {
return err return err
@ -834,6 +866,7 @@ func autoConvert_v1beta1_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta1.Sta
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }
@ -853,6 +886,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1beta1_StatefulSetSpec(in *apps.Statef
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }

View File

@ -20,6 +20,8 @@ import (
appsv1beta2 "k8s.io/api/apps/v1beta2" appsv1beta2 "k8s.io/api/apps/v1beta2"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
) )
func addDefaultingFuncs(scheme *runtime.Scheme) error { func addDefaultingFuncs(scheme *runtime.Scheme) error {
@ -72,6 +74,18 @@ func SetDefaults_StatefulSet(obj *appsv1beta2.StatefulSet) {
*obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0 *obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
obj.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = appsv1beta2.RetainPersistentVolumeClaimRetentionPolicyType
}
if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 {
obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = appsv1beta2.RetainPersistentVolumeClaimRetentionPolicyType
}
}
if obj.Spec.Replicas == nil { if obj.Spec.Replicas == nil {
obj.Spec.Replicas = new(int32) obj.Spec.Replicas = new(int32)
*obj.Spec.Replicas = 1 *obj.Spec.Replicas = 1

View File

@ -282,6 +282,16 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil { }); err != nil {
return err return err
} }
if err := s.AddGeneratedConversionFunc((*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), (*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(a.(*apps.StatefulSetPersistentVolumeClaimRetentionPolicy), b.(*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta2.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { if err := s.AddGeneratedConversionFunc((*v1beta2.StatefulSetStatus)(nil), (*apps.StatefulSetStatus)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta2_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta2.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope) return Convert_v1beta2_StatefulSetStatus_To_apps_StatefulSetStatus(a.(*v1beta2.StatefulSetStatus), b.(*apps.StatefulSetStatus), scope)
}); err != nil { }); err != nil {
@ -1250,6 +1260,28 @@ func Convert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in *apps.StatefulSe
return autoConvert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in, out, s) return autoConvert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in, out, s)
} }
func autoConvert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = apps.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, out *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy_To_apps_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
out.WhenDeleted = v1beta2.PersistentVolumeClaimRetentionPolicyType(in.WhenDeleted)
out.WhenScaled = v1beta2.PersistentVolumeClaimRetentionPolicyType(in.WhenScaled)
return nil
}
// Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy is an autogenerated conversion function.
func Convert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, out *v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy, s conversion.Scope) error {
return autoConvert_apps_StatefulSetPersistentVolumeClaimRetentionPolicy_To_v1beta2_StatefulSetPersistentVolumeClaimRetentionPolicy(in, out, s)
}
func autoConvert_v1beta2_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta2.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error { func autoConvert_v1beta2_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta2.StatefulSetSpec, out *apps.StatefulSetSpec, s conversion.Scope) error {
if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil { if err := metav1.Convert_Pointer_int32_To_int32(&in.Replicas, &out.Replicas, s); err != nil {
return err return err
@ -1266,6 +1298,7 @@ func autoConvert_v1beta2_StatefulSetSpec_To_apps_StatefulSetSpec(in *v1beta2.Sta
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*apps.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }
@ -1285,6 +1318,7 @@ func autoConvert_apps_StatefulSetSpec_To_v1beta2_StatefulSetSpec(in *apps.Statef
} }
out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit)) out.RevisionHistoryLimit = (*int32)(unsafe.Pointer(in.RevisionHistoryLimit))
out.MinReadySeconds = in.MinReadySeconds out.MinReadySeconds = in.MinReadySeconds
out.PersistentVolumeClaimRetentionPolicy = (*v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy)(unsafe.Pointer(in.PersistentVolumeClaimRetentionPolicy))
return nil return nil
} }

View File

@ -69,6 +69,26 @@ func ValidatePodTemplateSpecForStatefulSet(template *api.PodTemplateSpec, select
return allErrs return allErrs
} }
func ValidatePersistentVolumeClaimRetentionPolicyType(policy apps.PersistentVolumeClaimRetentionPolicyType, fldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
switch policy {
case apps.RetainPersistentVolumeClaimRetentionPolicyType:
case apps.DeletePersistentVolumeClaimRetentionPolicyType:
default:
allErrs = append(allErrs, field.NotSupported(fldPath, policy, []string{string(apps.RetainPersistentVolumeClaimRetentionPolicyType), string(apps.DeletePersistentVolumeClaimRetentionPolicyType)}))
}
return allErrs
}
func ValidatePersistentVolumeClaimRetentionPolicy(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy, fldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
if policy != nil {
allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicyType(policy.WhenDeleted, fldPath.Child("whenDeleted"))...)
allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicyType(policy.WhenScaled, fldPath.Child("whenScaled"))...)
}
return allErrs
}
// ValidateStatefulSetSpec tests if required fields in the StatefulSet spec are set. // ValidateStatefulSetSpec tests if required fields in the StatefulSet spec are set.
func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path, opts apivalidation.PodValidationOptions) field.ErrorList { func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path, opts apivalidation.PodValidationOptions) field.ErrorList {
allErrs := field.ErrorList{} allErrs := field.ErrorList{}
@ -108,6 +128,8 @@ func ValidateStatefulSetSpec(spec *apps.StatefulSetSpec, fldPath *field.Path, op
apps.OnDeleteStatefulSetStrategyType))) apps.OnDeleteStatefulSetStrategyType)))
} }
allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicy(spec.PersistentVolumeClaimRetentionPolicy, fldPath.Child("persistentVolumeClaimRetentionPolicy"))...)
allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.Replicas), fldPath.Child("replicas"))...) allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.Replicas), fldPath.Child("replicas"))...)
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) {
allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.MinReadySeconds), fldPath.Child("minReadySeconds"))...) allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(spec.MinReadySeconds), fldPath.Child("minReadySeconds"))...)
@ -158,11 +180,12 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) fi
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) {
newStatefulSetClone.Spec.MinReadySeconds = oldStatefulSet.Spec.MinReadySeconds // +k8s:verify-mutation:reason=clone newStatefulSetClone.Spec.MinReadySeconds = oldStatefulSet.Spec.MinReadySeconds // +k8s:verify-mutation:reason=clone
} }
newStatefulSetClone.Spec.PersistentVolumeClaimRetentionPolicy = oldStatefulSet.Spec.PersistentVolumeClaimRetentionPolicy // +k8s:verify-mutation:reason=clone
if !apiequality.Semantic.DeepEqual(newStatefulSetClone.Spec, oldStatefulSet.Spec) { if !apiequality.Semantic.DeepEqual(newStatefulSetClone.Spec, oldStatefulSet.Spec) {
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'minReadySeconds' and 'updateStrategy' are forbidden")) allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"))
} else { } else {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy' and 'minReadySeconds' are forbidden")) allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "updates to statefulset spec for fields other than 'replicas', 'template', 'updateStrategy' and 'persistentVolumeClaimRetentionPolicy' are forbidden"))
} }
} }
@ -170,6 +193,7 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) fi
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) { if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetMinReadySeconds) {
allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.MinReadySeconds), field.NewPath("spec", "minReadySeconds"))...) allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.MinReadySeconds), field.NewPath("spec", "minReadySeconds"))...)
} }
allErrs = append(allErrs, ValidatePersistentVolumeClaimRetentionPolicy(statefulSet.Spec.PersistentVolumeClaimRetentionPolicy, field.NewPath("spec", "persistentVolumeClaimRetentionPolicy"))...)
return allErrs return allErrs
} }

View File

@ -76,8 +76,10 @@ func TestValidateStatefulSet(t *testing.T) {
}, },
} }
successCases := []apps.StatefulSet{ const enableStatefulSetAutoDeletePVC = "[enable StatefulSetAutoDeletePVC]"
{
successCases := map[string]apps.StatefulSet{
"alpha name": {
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{ Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement, PodManagementPolicy: apps.OrderedReadyPodManagement,
@ -86,7 +88,7 @@ func TestValidateStatefulSet(t *testing.T) {
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
}, },
}, },
{ "alphanumeric name": {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{ Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement, PodManagementPolicy: apps.OrderedReadyPodManagement,
@ -95,7 +97,7 @@ func TestValidateStatefulSet(t *testing.T) {
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
}, },
}, },
{ "parallel pod management": {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{ Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.ParallelPodManagement, PodManagementPolicy: apps.ParallelPodManagement,
@ -104,7 +106,7 @@ func TestValidateStatefulSet(t *testing.T) {
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
}, },
}, },
{ "ordered ready pod management": {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{ Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement, PodManagementPolicy: apps.OrderedReadyPodManagement,
@ -113,7 +115,7 @@ func TestValidateStatefulSet(t *testing.T) {
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType},
}, },
}, },
{ "update strategy": {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault}, ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{ Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement, PodManagementPolicy: apps.OrderedReadyPodManagement,
@ -127,11 +129,29 @@ func TestValidateStatefulSet(t *testing.T) {
}()}, }()},
}, },
}, },
"PVC policy " + enableStatefulSetAutoDeletePVC: {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
},
},
} }
for i, successCase := range successCases { for name, successCase := range successCases {
t.Run("success case "+strconv.Itoa(i), func(t *testing.T) { name := name
if errs := ValidateStatefulSet(&successCase, corevalidation.PodValidationOptions{}); len(errs) != 0 { set := successCase
t.Run("success case "+name, func(t *testing.T) {
if strings.Contains(name, enableStatefulSetAutoDeletePVC) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
}
if errs := ValidateStatefulSet(&set, corevalidation.PodValidationOptions{}); len(errs) != 0 {
t.Errorf("expected success: %v", errs) t.Errorf("expected success: %v", errs)
} }
}) })
@ -352,10 +372,35 @@ func TestValidateStatefulSet(t *testing.T) {
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
}, },
}, },
"empty PersistentVolumeClaimRetentionPolicy " + enableStatefulSetAutoDeletePVC: {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{},
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
},
},
"invalid PersistentVolumeClaimRetentionPolicy " + enableStatefulSetAutoDeletePVC: {
ObjectMeta: metav1.ObjectMeta{Name: "abc-123", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.PersistentVolumeClaimRetentionPolicyType("invalid-delete-policy"),
},
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
},
},
} }
for k, v := range errorCases { for k, v := range errorCases {
t.Run(k, func(t *testing.T) { t.Run(k, func(t *testing.T) {
if strings.Contains(k, enableStatefulSetAutoDeletePVC) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
}
errs := ValidateStatefulSet(&v, corevalidation.PodValidationOptions{}) errs := ValidateStatefulSet(&v, corevalidation.PodValidationOptions{})
if len(errs) == 0 { if len(errs) == 0 {
t.Errorf("expected failure for %s", k) t.Errorf("expected failure for %s", k)
@ -378,6 +423,9 @@ func TestValidateStatefulSet(t *testing.T) {
field != "spec.updateStrategy.rollingUpdate" && field != "spec.updateStrategy.rollingUpdate" &&
field != "spec.updateStrategy.rollingUpdate.partition" && field != "spec.updateStrategy.rollingUpdate.partition" &&
field != "spec.podManagementPolicy" && field != "spec.podManagementPolicy" &&
field != "spec.persistentVolumeClaimRetentionPolicy" &&
field != "spec.persistentVolumeClaimRetentionPolicy.whenDeleted" &&
field != "spec.persistentVolumeClaimRetentionPolicy.whenScaled" &&
field != "spec.template.spec.activeDeadlineSeconds" { field != "spec.template.spec.activeDeadlineSeconds" {
t.Errorf("%s: missing prefix for: %v", k, errs[i]) t.Errorf("%s: missing prefix for: %v", k, errs[i])
} }
@ -748,6 +796,53 @@ func TestValidateStatefulSetUpdate(t *testing.T) {
}, },
}, },
}, },
{
old: apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: addContainersValidTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
},
},
update: apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
},
},
},
{
old: apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: addContainersValidTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
},
},
update: apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: &metav1.LabelSelector{MatchLabels: validLabels},
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
},
},
},
} }
for i, successCase := range successCases { for i, successCase := range successCases {

View File

@ -712,6 +712,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object {
return nil return nil
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy {
if in == nil {
return nil
}
out := new(StatefulSetPersistentVolumeClaimRetentionPolicy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = *in *out = *in
@ -734,6 +750,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = new(int32) *out = new(int32)
**out = **in **out = **in
} }
if in.PersistentVolumeClaimRetentionPolicy != nil {
in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy
*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
**out = **in
}
return return
} }

View File

@ -27,68 +27,115 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
errorutils "k8s.io/apimachinery/pkg/util/errors" errorutils "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
appslisters "k8s.io/client-go/listers/apps/v1"
corelisters "k8s.io/client-go/listers/core/v1" corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry" "k8s.io/client-go/util/retry"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
) )
// StatefulPodControlInterface defines the interface that StatefulSetController uses to create, update, and delete Pods, // StatefulPodControlObjectManager abstracts the manipulation of Pods and PVCs. The real controller implements this
// with a clientset for writes and listers for reads; for tests we provide stubs.
type StatefulPodControlObjectManager interface {
CreatePod(ctx context.Context, pod *v1.Pod) error
GetPod(namespace, podName string) (*v1.Pod, error)
UpdatePod(pod *v1.Pod) error
DeletePod(pod *v1.Pod) error
CreateClaim(claim *v1.PersistentVolumeClaim) error
GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error)
UpdateClaim(claim *v1.PersistentVolumeClaim) error
}
// StatefulPodControl defines the interface that StatefulSetController uses to create, update, and delete Pods,
// and to update the Status of a StatefulSet. It follows the design paradigms used for PodControl, but its // and to update the Status of a StatefulSet. It follows the design paradigms used for PodControl, but its
// implementation provides for PVC creation, ordered Pod creation, ordered Pod termination, and Pod identity enforcement. // implementation provides for PVC creation, ordered Pod creation, ordered Pod termination, and Pod identity enforcement.
// Like controller.PodControlInterface, it is implemented as an interface to provide for testing fakes. // Manipulation of objects is provided through objectMgr, which allows the k8s API to be mocked out for testing.
type StatefulPodControlInterface interface { type StatefulPodControl struct {
// CreateStatefulPod create a Pod in a StatefulSet. Any PVCs necessary for the Pod are created prior to creating objectMgr StatefulPodControlObjectManager
// the Pod. If the returned error is nil the Pod and its PVCs have been created.
CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error
// UpdateStatefulPod Updates a Pod in a StatefulSet. If the Pod already has the correct identity and stable
// storage this method is a no-op. If the Pod must be mutated to conform to the Set, it is mutated and updated.
// pod is an in-out parameter, and any updates made to the pod are reflected as mutations to this parameter. If
// the create is successful, the returned error is nil.
UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error
// DeleteStatefulPod deletes a Pod in a StatefulSet. The pods PVCs are not deleted. If the delete is successful,
// the returned error is nil.
DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error
}
func NewRealStatefulPodControl(
client clientset.Interface,
setLister appslisters.StatefulSetLister,
podLister corelisters.PodLister,
pvcLister corelisters.PersistentVolumeClaimLister,
recorder record.EventRecorder,
) StatefulPodControlInterface {
return &realStatefulPodControl{client, setLister, podLister, pvcLister, recorder}
}
// realStatefulPodControl implements StatefulPodControlInterface using a clientset.Interface to communicate with the
// API server. The struct is package private as the internal details are irrelevant to importing packages.
type realStatefulPodControl struct {
client clientset.Interface
setLister appslisters.StatefulSetLister
podLister corelisters.PodLister
pvcLister corelisters.PersistentVolumeClaimLister
recorder record.EventRecorder recorder record.EventRecorder
} }
func (spc *realStatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error { // NewStatefulPodControl constructs a StatefulPodControl using a realStatefulPodControlObjectManager with the given
// clientset, listers and EventRecorder.
func NewStatefulPodControl(
client clientset.Interface,
podLister corelisters.PodLister,
claimLister corelisters.PersistentVolumeClaimLister,
recorder record.EventRecorder,
) *StatefulPodControl {
return &StatefulPodControl{&realStatefulPodControlObjectManager{client, podLister, claimLister}, recorder}
}
// NewStatefulPodControlFromManager creates a StatefulPodControl using the given StatefulPodControlObjectManager and recorder.
func NewStatefulPodControlFromManager(om StatefulPodControlObjectManager, recorder record.EventRecorder) *StatefulPodControl {
return &StatefulPodControl{om, recorder}
}
// realStatefulPodControlObjectManager uses a clientset.Interface and listers.
type realStatefulPodControlObjectManager struct {
client clientset.Interface
podLister corelisters.PodLister
claimLister corelisters.PersistentVolumeClaimLister
}
func (om *realStatefulPodControlObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error {
_, err := om.client.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
return err
}
func (om *realStatefulPodControlObjectManager) GetPod(namespace, podName string) (*v1.Pod, error) {
return om.podLister.Pods(namespace).Get(podName)
}
func (om *realStatefulPodControlObjectManager) UpdatePod(pod *v1.Pod) error {
_, err := om.client.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{})
return err
}
func (om *realStatefulPodControlObjectManager) DeletePod(pod *v1.Pod) error {
return om.client.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
}
func (om *realStatefulPodControlObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim) error {
_, err := om.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), claim, metav1.CreateOptions{})
return err
}
func (om *realStatefulPodControlObjectManager) GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) {
return om.claimLister.PersistentVolumeClaims(namespace).Get(claimName)
}
func (om *realStatefulPodControlObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim) error {
_, err := om.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Update(context.TODO(), claim, metav1.UpdateOptions{})
return err
}
func (spc *StatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error {
// Create the Pod's PVCs prior to creating the Pod // Create the Pod's PVCs prior to creating the Pod
if err := spc.createPersistentVolumeClaims(set, pod); err != nil { if err := spc.createPersistentVolumeClaims(set, pod); err != nil {
spc.recordPodEvent("create", set, pod, err) spc.recordPodEvent("create", set, pod, err)
return err return err
} }
// If we created the PVCs attempt to create the Pod // If we created the PVCs attempt to create the Pod
_, err := spc.client.CoreV1().Pods(set.Namespace).Create(ctx, pod, metav1.CreateOptions{}) err := spc.objectMgr.CreatePod(ctx, pod)
// sink already exists errors // sink already exists errors
if apierrors.IsAlreadyExists(err) { if apierrors.IsAlreadyExists(err) {
return err return err
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
// Set PVC policy as much as is possible at this point.
if err := spc.UpdatePodClaimForRetentionPolicy(set, pod); err != nil {
spc.recordPodEvent("update", set, pod, err)
return err
}
}
spc.recordPodEvent("create", set, pod, err) spc.recordPodEvent("create", set, pod, err)
return err return err
} }
func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { func (spc *StatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error {
attemptedUpdate := false attemptedUpdate := false
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
// assume the Pod is consistent // assume the Pod is consistent
@ -108,6 +155,21 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod
return err return err
} }
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
// if the Pod's PVCs are not consistent with the StatefulSet's PVC deletion policy, update the PVC
// and dirty the pod.
if match, err := spc.ClaimsMatchRetentionPolicy(set, pod); err != nil {
spc.recordPodEvent("update", set, pod, err)
return err
} else if !match {
if err := spc.UpdatePodClaimForRetentionPolicy(set, pod); err != nil {
spc.recordPodEvent("update", set, pod, err)
return err
}
consistent = false
}
}
// if the Pod is not dirty, do nothing // if the Pod is not dirty, do nothing
if consistent { if consistent {
return nil return nil
@ -115,16 +177,17 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod
attemptedUpdate = true attemptedUpdate = true
// commit the update, retrying on conflicts // commit the update, retrying on conflicts
_, updateErr := spc.client.CoreV1().Pods(set.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{})
updateErr := spc.objectMgr.UpdatePod(pod)
if updateErr == nil { if updateErr == nil {
return nil return nil
} }
if updated, err := spc.podLister.Pods(set.Namespace).Get(pod.Name); err == nil { if updated, err := spc.objectMgr.GetPod(set.Namespace, pod.Name); err == nil {
// make a copy so we don't mutate the shared cache // make a copy so we don't mutate the shared cache
pod = updated.DeepCopy() pod = updated.DeepCopy()
} else { } else {
utilruntime.HandleError(fmt.Errorf("error getting updated Pod %s/%s from lister: %v", set.Namespace, pod.Name, err)) utilruntime.HandleError(fmt.Errorf("error getting updated Pod %s/%s: %w", set.Namespace, pod.Name, err))
} }
return updateErr return updateErr
@ -135,15 +198,92 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod
return err return err
} }
func (spc *realStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { func (spc *StatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error {
err := spc.client.CoreV1().Pods(set.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) err := spc.objectMgr.DeletePod(pod)
spc.recordPodEvent("delete", set, pod, err) spc.recordPodEvent("delete", set, pod, err)
return err return err
} }
// ClaimsMatchRetentionPolicy returns false if the PVCs for pod are not consistent with set's PVC deletion policy.
// An error is returned if something is not consistent. This is expected if the pod is being otherwise updated,
// but a problem otherwise (see usage of this method in UpdateStatefulPod).
func (spc *StatefulPodControl) ClaimsMatchRetentionPolicy(set *apps.StatefulSet, pod *v1.Pod) (bool, error) {
ordinal := getOrdinal(pod)
templates := set.Spec.VolumeClaimTemplates
for i := range templates {
claimName := getPersistentVolumeClaimName(set, &templates[i], ordinal)
claim, err := spc.objectMgr.GetClaim(set.Namespace, claimName)
switch {
case apierrors.IsNotFound(err):
klog.V(4).Infof("Expected claim %s missing, continuing to pick up in next iteration", claimName)
case err != nil:
return false, fmt.Errorf("Could not retrieve claim %s for %s when checking PVC deletion policy", claimName, pod.Name)
default:
if !claimOwnerMatchesSetAndPod(claim, set, pod) {
return false, nil
}
}
}
return true, nil
}
// UpdatePodClaimForRetentionPolicy updates the PVCs used by pod to match the PVC deletion policy of set.
func (spc *StatefulPodControl) UpdatePodClaimForRetentionPolicy(set *apps.StatefulSet, pod *v1.Pod) error {
ordinal := getOrdinal(pod)
templates := set.Spec.VolumeClaimTemplates
for i := range templates {
claimName := getPersistentVolumeClaimName(set, &templates[i], ordinal)
claim, err := spc.objectMgr.GetClaim(set.Namespace, claimName)
switch {
case apierrors.IsNotFound(err):
klog.V(4).Infof("Expected claim %s missing, continuing to pick up in next iteration.")
case err != nil:
return fmt.Errorf("Could not retrieve claim %s not found for %s when checking PVC deletion policy: %w", claimName, pod.Name, err)
default:
if !claimOwnerMatchesSetAndPod(claim, set, pod) {
needsUpdate := updateClaimOwnerRefForSetAndPod(claim, set, pod)
if needsUpdate {
err := spc.objectMgr.UpdateClaim(claim)
if err != nil {
return fmt.Errorf("Could not update claim %s for delete policy ownerRefs: %w", claimName, err)
}
}
}
}
}
return nil
}
// PodClaimIsStale returns true for a stale PVC that should block pod creation. If the scaling
// policy is deletion, and a PVC has an ownerRef that does not match the pod, the PVC is stale. This
// includes pods whose UID has not been created.
func (spc *StatefulPodControl) PodClaimIsStale(set *apps.StatefulSet, pod *v1.Pod) (bool, error) {
policy := getPersistentVolumeClaimRetentionPolicy(set)
if policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType {
// PVCs are meant to be reused and so can't be stale.
return false, nil
}
for _, claim := range getPersistentVolumeClaims(set, pod) {
pvc, err := spc.objectMgr.GetClaim(claim.Namespace, claim.Name)
switch {
case apierrors.IsNotFound(err):
// If the claim doesn't exist yet, it can't be stale.
continue
case err != nil:
return false, err
case err == nil:
// A claim is stale if it doesn't match the pod's UID, including if the pod has no UID.
if hasStaleOwnerRef(pvc, pod) {
return true, nil
}
}
}
return false, nil
}
// recordPodEvent records an event for verb applied to a Pod in a StatefulSet. If err is nil the generated event will // recordPodEvent records an event for verb applied to a Pod in a StatefulSet. If err is nil the generated event will
// have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a reason of v1.EventTypeWarning. // have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a reason of v1.EventTypeWarning.
func (spc *realStatefulPodControl) recordPodEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, err error) { func (spc *StatefulPodControl) recordPodEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, err error) {
if err == nil { if err == nil {
reason := fmt.Sprintf("Successful%s", strings.Title(verb)) reason := fmt.Sprintf("Successful%s", strings.Title(verb))
message := fmt.Sprintf("%s Pod %s in StatefulSet %s successful", message := fmt.Sprintf("%s Pod %s in StatefulSet %s successful",
@ -160,7 +300,7 @@ func (spc *realStatefulPodControl) recordPodEvent(verb string, set *apps.Statefu
// recordClaimEvent records an event for verb applied to the PersistentVolumeClaim of a Pod in a StatefulSet. If err is // recordClaimEvent records an event for verb applied to the PersistentVolumeClaim of a Pod in a StatefulSet. If err is
// nil the generated event will have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a // nil the generated event will have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a
// reason of v1.EventTypeWarning. // reason of v1.EventTypeWarning.
func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, err error) { func (spc *StatefulPodControl) recordClaimEvent(verb string, set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, err error) {
if err == nil { if err == nil {
reason := fmt.Sprintf("Successful%s", strings.Title(verb)) reason := fmt.Sprintf("Successful%s", strings.Title(verb))
message := fmt.Sprintf("%s Claim %s Pod %s in StatefulSet %s success", message := fmt.Sprintf("%s Claim %s Pod %s in StatefulSet %s success",
@ -178,13 +318,13 @@ func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.State
// set. If all of the claims for Pod are successfully created, the returned error is nil. If creation fails, this method // set. If all of the claims for Pod are successfully created, the returned error is nil. If creation fails, this method
// may be called again until no error is returned, indicating the PersistentVolumeClaims for pod are consistent with // may be called again until no error is returned, indicating the PersistentVolumeClaims for pod are consistent with
// set's Spec. // set's Spec.
func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { func (spc *StatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error {
var errs []error var errs []error
for _, claim := range getPersistentVolumeClaims(set, pod) { for _, claim := range getPersistentVolumeClaims(set, pod) {
pvc, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) pvc, err := spc.objectMgr.GetClaim(claim.Namespace, claim.Name)
switch { switch {
case apierrors.IsNotFound(err): case apierrors.IsNotFound(err):
_, err := spc.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), &claim, metav1.CreateOptions{}) err := spc.objectMgr.CreateClaim(&claim)
if err != nil { if err != nil {
errs = append(errs, fmt.Errorf("failed to create PVC %s: %s", claim.Name, err)) errs = append(errs, fmt.Errorf("failed to create PVC %s: %s", claim.Name, err))
} }
@ -203,5 +343,3 @@ func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.Statef
} }
return errorutils.NewAggregate(errs) return errorutils.NewAggregate(errs)
} }
var _ StatefulPodControlInterface = &realStatefulPodControl{}

View File

@ -19,23 +19,27 @@ package statefulset
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"strings" "strings"
"testing" "testing"
"time" "time"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
corelisters "k8s.io/client-go/listers/core/v1"
core "k8s.io/client-go/testing" core "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
corelisters "k8s.io/client-go/listers/core/v1"
_ "k8s.io/kubernetes/pkg/apis/apps/install" _ "k8s.io/kubernetes/pkg/apis/apps/install"
_ "k8s.io/kubernetes/pkg/apis/core/install" _ "k8s.io/kubernetes/pkg/apis/core/install"
"k8s.io/kubernetes/pkg/features"
) )
func TestStatefulPodControlCreatesPods(t *testing.T) { func TestStatefulPodControlCreatesPods(t *testing.T) {
@ -43,14 +47,15 @@ func TestStatefulPodControlCreatesPods(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder)
fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource) return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource)
}) })
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
create := action.(core.CreateAction) create := action.(core.CreateAction)
claimIndexer.Add(create.GetObject())
return true, create.GetObject(), nil return true, create.GetObject(), nil
}) })
fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
@ -83,7 +88,7 @@ func TestStatefulPodControlCreatePodExists(t *testing.T) {
pvcIndexer.Add(&pvc) pvcIndexer.Add(&pvc)
} }
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
create := action.(core.CreateAction) create := action.(core.CreateAction)
return true, create.GetObject(), nil return true, create.GetObject(), nil
@ -110,7 +115,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) {
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewInternalError(errors.New("API server down")) return true, nil, apierrors.NewInternalError(errors.New("API server down"))
}) })
@ -131,7 +136,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) {
} }
} }
} }
func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) { func TestStatefulPodControlCreatePodPVCDeleting(t *testing.T) {
recorder := record.NewFakeRecorder(10) recorder := record.NewFakeRecorder(10)
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
@ -145,7 +150,7 @@ func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) {
pvcIndexer.Add(&pvc) pvcIndexer.Add(&pvc)
} }
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
create := action.(core.CreateAction) create := action.(core.CreateAction)
return true, create.GetObject(), nil return true, create.GetObject(), nil
@ -184,7 +189,7 @@ func TestStatefulPodControlCreatePodPvcGetFailure(t *testing.T) {
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := &fakeIndexer{getError: errors.New("API server down")} pvcIndexer := &fakeIndexer{getError: errors.New("API server down")}
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewInternalError(errors.New("API server down")) return true, nil, apierrors.NewInternalError(errors.New("API server down"))
}) })
@ -213,7 +218,7 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) {
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
create := action.(core.CreateAction) create := action.(core.CreateAction)
return true, create.GetObject(), nil return true, create.GetObject(), nil
@ -232,7 +237,6 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) {
} else if !strings.Contains(events[1], v1.EventTypeWarning) { } else if !strings.Contains(events[1], v1.EventTypeWarning) {
t.Errorf("Found unexpected non-warning event %s", events[1]) t.Errorf("Found unexpected non-warning event %s", events[1])
} }
} }
@ -241,7 +245,14 @@ func TestStatefulPodControlNoOpUpdate(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) claims := getPersistentVolumeClaims(set, pod)
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
for k := range claims {
claim := claims[k]
indexer.Add(&claim)
}
claimLister := corelisters.NewPersistentVolumeClaimLister(indexer)
control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder)
fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) {
t.Error("no-op update should not make any client invocation") t.Error("no-op update should not make any client invocation")
return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem")) return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem"))
@ -260,7 +271,9 @@ func TestStatefulPodControlUpdatesIdentity(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := fake.NewSimpleClientset(set, pod) fakeClient := fake.NewSimpleClientset(set, pod)
control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(indexer)
control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder)
var updated *v1.Pod var updated *v1.Pod
fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) {
update := action.(core.UpdateAction) update := action.(core.UpdateAction)
@ -287,12 +300,14 @@ func TestStatefulPodControlUpdateIdentityFailure(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
gooPod := newStatefulSetPod(set, 0) gooPod := newStatefulSetPod(set, 0)
gooPod.Name = "goo-0" gooPod.Name = "goo-0"
indexer.Add(gooPod) podIndexer.Add(gooPod)
podLister := corelisters.NewPodLister(indexer) podLister := corelisters.NewPodLister(podIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder)
fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) {
pod.Name = "goo-0" pod.Name = "goo-0"
return true, nil, apierrors.NewInternalError(errors.New("API server down")) return true, nil, apierrors.NewInternalError(errors.New("API server down"))
@ -319,7 +334,7 @@ func TestStatefulPodControlUpdatesPodStorage(t *testing.T) {
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
pvcs := getPersistentVolumeClaims(set, pod) pvcs := getPersistentVolumeClaims(set, pod)
volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes))
for i := range pod.Spec.Volumes { for i := range pod.Spec.Volumes {
@ -366,7 +381,7 @@ func TestStatefulPodControlUpdatePodStorageFailure(t *testing.T) {
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder)
pvcs := getPersistentVolumeClaims(set, pod) pvcs := getPersistentVolumeClaims(set, pod)
volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes))
for i := range pod.Spec.Volumes { for i := range pod.Spec.Volumes {
@ -401,12 +416,19 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
podLister := corelisters.NewPodLister(podIndexer)
claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(podIndexer)
gooPod := newStatefulSetPod(set, 0) gooPod := newStatefulSetPod(set, 0)
gooPod.Name = "goo-0" gooPod.Labels[apps.StatefulSetPodNameLabel] = "goo-starts"
indexer.Add(gooPod) podIndexer.Add(gooPod)
podLister := corelisters.NewPodLister(indexer) claims := getPersistentVolumeClaims(set, gooPod)
control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) for k := range claims {
claim := claims[k]
claimIndexer.Add(&claim)
}
control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder)
conflict := false conflict := false
fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) {
update := action.(core.UpdateAction) update := action.(core.UpdateAction)
@ -417,7 +439,7 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) {
return true, update.GetObject(), nil return true, update.GetObject(), nil
}) })
pod.Name = "goo-0" pod.Labels[apps.StatefulSetPodNameLabel] = "goo-0"
if err := control.UpdateStatefulPod(set, pod); err != nil { if err := control.UpdateStatefulPod(set, pod); err != nil {
t.Errorf("Successful update returned an error: %s", err) t.Errorf("Successful update returned an error: %s", err)
} }
@ -437,7 +459,7 @@ func TestStatefulPodControlDeletesStatefulPod(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) control := NewStatefulPodControl(fakeClient, nil, nil, recorder)
fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil return true, nil, nil
}) })
@ -457,7 +479,7 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{} fakeClient := &fake.Clientset{}
control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) control := NewStatefulPodControl(fakeClient, nil, nil, recorder)
fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewInternalError(errors.New("API server down")) return true, nil, apierrors.NewInternalError(errors.New("API server down"))
}) })
@ -472,6 +494,344 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) {
} }
} }
func TestStatefulPodControlClaimsMatchDeletionPolcy(t *testing.T) {
// The claimOwnerMatchesSetAndPod is tested exhaustively in stateful_set_utils_test; this
// test is for the wiring to the method tested there.
fakeClient := &fake.Clientset{}
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(indexer)
set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0)
claims := getPersistentVolumeClaims(set, pod)
for k := range claims {
claim := claims[k]
indexer.Add(&claim)
}
control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{})
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil {
t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (retain): %v", err)
} else if !matches {
t.Error("Unexpected non-match for ClaimsMatchRetentionPolicy (retain)")
}
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil {
t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (set deletion): %v", err)
} else if matches {
t.Error("Unexpected match for ClaimsMatchRetentionPolicy (set deletion)")
}
}
func TestStatefulPodControlUpdatePodClaimForRetentionPolicy(t *testing.T) {
// All the update conditions are tested exhaustively in stateful_set_utils_test. This
// tests the wiring from the pod control to that method.
testFn := func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
fakeClient := &fake.Clientset{}
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(indexer)
set := newStatefulSet(3)
set.GetObjectMeta().SetUID("set-123")
pod := newStatefulSetPod(set, 0)
claims := getPersistentVolumeClaims(set, pod)
for k := range claims {
claim := claims[k]
indexer.Add(&claim)
}
control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{})
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
if err := control.UpdatePodClaimForRetentionPolicy(set, pod); err != nil {
t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err)
}
expectRef := utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC)
for k := range claims {
claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name)
if err != nil {
t.Errorf("Unexpected error getting Claim %s/%s: %v", claim.Namespace, claim.Name, err)
}
if hasOwnerRef(claim, set) != expectRef {
t.Errorf("Claim %s/%s bad set owner ref", claim.Namespace, claim.Name)
}
}
}
t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
testFn(t)
})
t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)()
testFn(t)
})
}
func TestPodClaimIsStale(t *testing.T) {
const missing = "missing"
const exists = "exists"
const stale = "stale"
const withRef = "with-ref"
testCases := []struct {
name string
claimStates []string
expected bool
skipPodUID bool
}{
{
name: "all missing",
claimStates: []string{missing, missing},
expected: false,
},
{
name: "no claims",
claimStates: []string{},
expected: false,
},
{
name: "exists",
claimStates: []string{missing, exists},
expected: false,
},
{
name: "all refs",
claimStates: []string{withRef, withRef},
expected: false,
},
{
name: "stale & exists",
claimStates: []string{stale, exists},
expected: true,
},
{
name: "stale & missing",
claimStates: []string{stale, missing},
expected: true,
},
{
name: "withRef & stale",
claimStates: []string{withRef, stale},
expected: true,
},
{
name: "withRef, no UID",
claimStates: []string{withRef},
skipPodUID: true,
expected: true,
},
}
for _, tc := range testCases {
set := apps.StatefulSet{}
set.Name = "set"
set.Namespace = "default"
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
}
set.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"key": "value"}}
claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
for i, claimState := range tc.claimStates {
claim := v1.PersistentVolumeClaim{}
claim.Name = fmt.Sprintf("claim-%d", i)
set.Spec.VolumeClaimTemplates = append(set.Spec.VolumeClaimTemplates, claim)
claim.Name = fmt.Sprintf("%s-set-3", claim.Name)
claim.Namespace = set.Namespace
switch claimState {
case missing:
// Do nothing, the claim shouldn't exist.
case exists:
claimIndexer.Add(&claim)
case stale:
claim.SetOwnerReferences([]metav1.OwnerReference{
{Name: "set-3", UID: types.UID("stale")},
})
claimIndexer.Add(&claim)
case withRef:
claim.SetOwnerReferences([]metav1.OwnerReference{
{Name: "set-3", UID: types.UID("123")},
})
claimIndexer.Add(&claim)
}
}
pod := v1.Pod{}
pod.Name = "set-3"
if !tc.skipPodUID {
pod.SetUID("123")
}
claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
control := NewStatefulPodControl(&fake.Clientset{}, nil, claimLister, &noopRecorder{})
expected := tc.expected
// Note that the error isn't / can't be tested.
if stale, _ := control.PodClaimIsStale(&set, &pod); stale != expected {
t.Errorf("unexpected stale for %s", tc.name)
}
}
}
func TestStatefulPodControlRetainDeletionPolicyUpdate(t *testing.T) {
testFn := func(t *testing.T) {
recorder := record.NewFakeRecorder(10)
set := newStatefulSet(1)
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{}
podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
podLister := corelisters.NewPodLister(podIndexer)
claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
podIndexer.Add(pod)
claims := getPersistentVolumeClaims(set, pod)
if len(claims) < 1 {
t.Errorf("Unexpected missing PVCs")
}
for k := range claims {
claim := claims[k]
setOwnerRef(&claim, set, &set.TypeMeta) // This ownerRef should be removed in the update.
claimIndexer.Add(&claim)
}
control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder)
if err := control.UpdateStatefulPod(set, pod); err != nil {
t.Errorf("Successful update returned an error: %s", err)
}
for k := range claims {
claim := claims[k]
if hasOwnerRef(&claim, set) {
t.Errorf("ownerRef not removed: %s/%s", claim.Namespace, claim.Name)
}
}
events := collectEvents(recorder.Events)
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if eventCount := len(events); eventCount != 1 {
t.Errorf("delete failed: got %d events, but want 1", eventCount)
}
} else {
if len(events) != 0 {
t.Errorf("delete failed: expected no events, but got %v", events)
}
}
}
t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
testFn(t)
})
t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)()
testFn(t)
})
}
func TestStatefulPodControlRetentionPolicyUpdate(t *testing.T) {
// Only applicable when the feature gate is on; the off case is tested in TestStatefulPodControlRetainRetentionPolicyUpdate.
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
recorder := record.NewFakeRecorder(10)
set := newStatefulSet(1)
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{}
podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
podIndexer.Add(pod)
claims := getPersistentVolumeClaims(set, pod)
if len(claims) != 1 {
t.Errorf("Unexpected or missing PVCs")
}
var claim v1.PersistentVolumeClaim
for k := range claims {
claim = claims[k]
claimIndexer.Add(&claim)
}
fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
update := action.(core.UpdateAction)
claimIndexer.Update(update.GetObject())
return true, update.GetObject(), nil
})
podLister := corelisters.NewPodLister(podIndexer)
claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder)
if err := control.UpdateStatefulPod(set, pod); err != nil {
t.Errorf("Successful update returned an error: %s", err)
}
updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
if err != nil {
t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err)
}
if !hasOwnerRef(updatedClaim, set) {
t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name)
}
events := collectEvents(recorder.Events)
if eventCount := len(events); eventCount != 1 {
t.Errorf("update failed: got %d events, but want 1", eventCount)
}
}
func TestStatefulPodControlRetentionPolicyUpdateMissingClaims(t *testing.T) {
// Only applicable when the feature gate is on.
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
recorder := record.NewFakeRecorder(10)
set := newStatefulSet(1)
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
pod := newStatefulSetPod(set, 0)
fakeClient := &fake.Clientset{}
podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
podLister := corelisters.NewPodLister(podIndexer)
claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer)
podIndexer.Add(pod)
fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
update := action.(core.UpdateAction)
claimIndexer.Update(update.GetObject())
return true, update.GetObject(), nil
})
control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder)
if err := control.UpdateStatefulPod(set, pod); err != nil {
t.Error("Unexpected error on pod update when PVCs are missing")
}
claims := getPersistentVolumeClaims(set, pod)
if len(claims) != 1 {
t.Errorf("Unexpected or missing PVCs")
}
var claim v1.PersistentVolumeClaim
for k := range claims {
claim = claims[k]
claimIndexer.Add(&claim)
}
if err := control.UpdateStatefulPod(set, pod); err != nil {
t.Errorf("Expected update to succeed, saw error %v", err)
}
updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
if err != nil {
t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err)
}
if !hasOwnerRef(updatedClaim, set) {
t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name)
}
events := collectEvents(recorder.Events)
if eventCount := len(events); eventCount != 1 {
t.Errorf("update failed: got %d events, but want 2", eventCount)
}
if !strings.Contains(events[0], "SuccessfulUpdate") {
t.Errorf("expected first event to be a successful update: %s", events[1])
}
}
func collectEvents(source <-chan string) []string { func collectEvents(source <-chan string) []string {
done := false done := false
events := make([]string, 0) events := make([]string, 0)

View File

@ -91,9 +91,8 @@ func NewStatefulSetController(
ssc := &StatefulSetController{ ssc := &StatefulSetController{
kubeClient: kubeClient, kubeClient: kubeClient,
control: NewDefaultStatefulSetControl( control: NewDefaultStatefulSetControl(
NewRealStatefulPodControl( NewStatefulPodControl(
kubeClient, kubeClient,
setInformer.Lister(),
podInformer.Lister(), podInformer.Lister(),
pvcInformer.Lister(), pvcInformer.Lister(),
recorder), recorder),
@ -352,10 +351,14 @@ func (ssc *StatefulSetController) getStatefulSetsForPod(pod *v1.Pod) []*apps.Sta
if len(sets) > 1 { if len(sets) > 1 {
// ControllerRef will ensure we don't do anything crazy, but more than one // ControllerRef will ensure we don't do anything crazy, but more than one
// item in this list nevertheless constitutes user error. // item in this list nevertheless constitutes user error.
setNames := []string{}
for _, s := range sets {
setNames = append(setNames, s.Name)
}
utilruntime.HandleError( utilruntime.HandleError(
fmt.Errorf( fmt.Errorf(
"user error: more than one StatefulSet is selecting pods with labels: %+v", "user error: more than one StatefulSet is selecting pods with labels: %+v. Sets: %v",
pod.Labels)) pod.Labels, setNames))
} }
return sets return sets
} }

View File

@ -54,7 +54,7 @@ type StatefulSetControlInterface interface {
// to update the status of StatefulSets. You should use an instance returned from NewRealStatefulPodControl() for any // to update the status of StatefulSets. You should use an instance returned from NewRealStatefulPodControl() for any
// scenario other than testing. // scenario other than testing.
func NewDefaultStatefulSetControl( func NewDefaultStatefulSetControl(
podControl StatefulPodControlInterface, podControl *StatefulPodControl,
statusUpdater StatefulSetStatusUpdaterInterface, statusUpdater StatefulSetStatusUpdaterInterface,
controllerHistory history.Interface, controllerHistory history.Interface,
recorder record.EventRecorder) StatefulSetControlInterface { recorder record.EventRecorder) StatefulSetControlInterface {
@ -62,7 +62,7 @@ func NewDefaultStatefulSetControl(
} }
type defaultStatefulSetControl struct { type defaultStatefulSetControl struct {
podControl StatefulPodControlInterface podControl *StatefulPodControl
statusUpdater StatefulSetStatusUpdaterInterface statusUpdater StatefulSetStatusUpdaterInterface
controllerHistory history.Interface controllerHistory history.Interface
recorder record.EventRecorder recorder record.EventRecorder
@ -333,7 +333,6 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
// if the ordinal of the pod is within the range of the current number of replicas, // if the ordinal of the pod is within the range of the current number of replicas,
// insert it at the indirection of its ordinal // insert it at the indirection of its ordinal
replicas[ord] = pods[i] replicas[ord] = pods[i]
} else if ord >= replicaCount { } else if ord >= replicaCount {
// if the ordinal is greater than the number of replicas add it to the condemned list // if the ordinal is greater than the number of replicas add it to the condemned list
condemned = append(condemned, pods[i]) condemned = append(condemned, pods[i])
@ -418,6 +417,14 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
} }
// If we find a Pod that has not been created we create the Pod // If we find a Pod that has not been created we create the Pod
if !isCreated(replicas[i]) { if !isCreated(replicas[i]) {
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if isStale, err := ssc.podControl.PodClaimIsStale(set, replicas[i]); err != nil {
return &status, err
} else if isStale {
// If a pod has a stale PVC, no more work can be done this round.
return &status, err
}
}
if err := ssc.podControl.CreateStatefulPod(ctx, set, replicas[i]); err != nil { if err := ssc.podControl.CreateStatefulPod(ctx, set, replicas[i]); err != nil {
return &status, err return &status, err
} }
@ -428,7 +435,6 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
if getPodRevision(replicas[i]) == updateRevision.Name { if getPodRevision(replicas[i]) == updateRevision.Name {
status.UpdatedReplicas++ status.UpdatedReplicas++
} }
// if the set does not allow bursting, return immediately // if the set does not allow bursting, return immediately
if monotonic { if monotonic {
return &status, nil return &status, nil
@ -471,7 +477,16 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
return &status, nil return &status, nil
} }
// Enforce the StatefulSet invariants // Enforce the StatefulSet invariants
if identityMatches(set, replicas[i]) && storageMatches(set, replicas[i]) { retentionMatch := true
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
var err error
retentionMatch, err = ssc.podControl.ClaimsMatchRetentionPolicy(updateSet, replicas[i])
// An error is expected if the pod is not yet fully updated, and so return is treated as matching.
if err != nil {
retentionMatch = true
}
}
if identityMatches(set, replicas[i]) && storageMatches(set, replicas[i]) && retentionMatch {
continue continue
} }
// Make a deep copy so we don't mutate the shared cache // Make a deep copy so we don't mutate the shared cache
@ -481,6 +496,19 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet(
} }
} }
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
// Ensure ownerRefs are set correctly for the condemned pods.
for i := range condemned {
if matchPolicy, err := ssc.podControl.ClaimsMatchRetentionPolicy(updateSet, condemned[i]); err != nil {
return &status, err
} else if !matchPolicy {
if err := ssc.podControl.UpdatePodClaimForRetentionPolicy(updateSet, condemned[i]); err != nil {
return &status, err
}
}
}
}
// At this point, all of the current Replicas are Running, Ready and Available, we can consider termination. // At this point, all of the current Replicas are Running, Ready and Available, we can consider termination.
// We will wait for all predecessors to be Running and Ready prior to attempting a deletion. // We will wait for all predecessors to be Running and Ready prior to attempting a deletion.
// We will terminate Pods in a monotonically decreasing order over [len(pods),set.Spec.Replicas). // We will terminate Pods in a monotonically decreasing order over [len(pods),set.Spec.Replicas).

File diff suppressed because it is too large Load Diff

View File

@ -20,21 +20,28 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"sort" "sort"
"testing" "testing"
apps "k8s.io/api/apps/v1" apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers" "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/history" "k8s.io/kubernetes/pkg/controller/history"
"k8s.io/kubernetes/pkg/features"
) )
var parentKind = apps.SchemeGroupVersion.WithKind("StatefulSet") var parentKind = apps.SchemeGroupVersion.WithKind("StatefulSet")
@ -43,11 +50,11 @@ func alwaysReady() bool { return true }
func TestStatefulSetControllerCreates(t *testing.T) { func TestStatefulSetControllerCreates(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, spc, om, _ := newFakeStatefulSetController(set)
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn up StatefulSet : %s", err) t.Errorf("Failed to turn up StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -59,11 +66,11 @@ func TestStatefulSetControllerCreates(t *testing.T) {
func TestStatefulSetControllerDeletes(t *testing.T) { func TestStatefulSetControllerDeletes(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, spc, om, _ := newFakeStatefulSetController(set)
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn up StatefulSet : %s", err) t.Errorf("Failed to turn up StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -72,10 +79,10 @@ func TestStatefulSetControllerDeletes(t *testing.T) {
t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas)
} }
*set.Spec.Replicas = 0 *set.Spec.Replicas = 0
if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn down StatefulSet : %s", err) t.Errorf("Failed to turn down StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -87,11 +94,11 @@ func TestStatefulSetControllerDeletes(t *testing.T) {
func TestStatefulSetControllerRespectsTermination(t *testing.T) { func TestStatefulSetControllerRespectsTermination(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, spc, om, _ := newFakeStatefulSetController(set)
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn up StatefulSet : %s", err) t.Errorf("Failed to turn up StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -99,11 +106,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) {
if set.Status.Replicas != 3 { if set.Status.Replicas != 3 {
t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas)
} }
_, err := spc.addTerminatingPod(set, 3) _, err := om.addTerminatingPod(set, 3)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
pods, err := spc.addTerminatingPod(set, 4) pods, err := om.addTerminatingPod(set, 4)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -112,7 +119,7 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) {
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
pods, err = spc.podsLister.Pods(set.Namespace).List(selector) pods, err = om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -123,10 +130,10 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) {
spc.DeleteStatefulPod(set, pods[3]) spc.DeleteStatefulPod(set, pods[3])
spc.DeleteStatefulPod(set, pods[4]) spc.DeleteStatefulPod(set, pods[4])
*set.Spec.Replicas = 0 *set.Spec.Replicas = 0
if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn down StatefulSet : %s", err) t.Errorf("Failed to turn down StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -138,11 +145,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) {
func TestStatefulSetControllerBlocksScaling(t *testing.T) { func TestStatefulSetControllerBlocksScaling(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, spc, om, _ := newFakeStatefulSetController(set)
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf("Failed to turn up StatefulSet : %s", err) t.Errorf("Failed to turn up StatefulSet : %s", err)
} }
if obj, _, err := spc.setsIndexer.Get(set); err != nil { if obj, _, err := om.setsIndexer.Get(set); err != nil {
t.Error(err) t.Error(err)
} else { } else {
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
@ -152,8 +159,8 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) {
} }
*set.Spec.Replicas = 5 *set.Spec.Replicas = 5
fakeResourceVersion(set) fakeResourceVersion(set)
spc.setsIndexer.Update(set) om.setsIndexer.Update(set)
_, err := spc.setPodTerminated(set, 0) _, err := om.setPodTerminated(set, 0)
if err != nil { if err != nil {
t.Error("Failed to set pod terminated at ordinal 0") t.Error("Failed to set pod terminated at ordinal 0")
} }
@ -163,7 +170,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) {
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
pods, err := spc.podsLister.Pods(set.Namespace).List(selector) pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -174,7 +181,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) {
spc.DeleteStatefulPod(set, pods[0]) spc.DeleteStatefulPod(set, pods[0])
ssc.enqueueStatefulSet(set) ssc.enqueueStatefulSet(set)
fakeWorker(ssc) fakeWorker(ssc)
pods, err = spc.podsLister.Pods(set.Namespace).List(selector) pods, err = om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -186,9 +193,9 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) {
func TestStatefulSetControllerDeletionTimestamp(t *testing.T) { func TestStatefulSetControllerDeletionTimestamp(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
set.DeletionTimestamp = new(metav1.Time) set.DeletionTimestamp = new(metav1.Time)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, _, om, _ := newFakeStatefulSetController(set)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
// Force a sync. It should not try to create any Pods. // Force a sync. It should not try to create any Pods.
ssc.enqueueStatefulSet(set) ssc.enqueueStatefulSet(set)
@ -198,7 +205,7 @@ func TestStatefulSetControllerDeletionTimestamp(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
pods, err := spc.podsLister.Pods(set.Namespace).List(selector) pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -211,17 +218,17 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
// The bare client says it IS deleted. // The bare client says it IS deleted.
set.DeletionTimestamp = new(metav1.Time) set.DeletionTimestamp = new(metav1.Time)
ssc, spc, ssh := newFakeStatefulSetController(set) ssc, _, om, ssh := newFakeStatefulSetController(set)
// The lister (cache) says it's NOT deleted. // The lister (cache) says it's NOT deleted.
set2 := *set set2 := *set
set2.DeletionTimestamp = nil set2.DeletionTimestamp = nil
spc.setsIndexer.Add(&set2) om.setsIndexer.Add(&set2)
// The recheck occurs in the presence of a matching orphan. // The recheck occurs in the presence of a matching orphan.
pod := newStatefulSetPod(set, 1) pod := newStatefulSetPod(set, 1)
pod.OwnerReferences = nil pod.OwnerReferences = nil
spc.podsIndexer.Add(pod) om.podsIndexer.Add(pod)
set.Status.CollisionCount = new(int32) set.Status.CollisionCount = new(int32)
revision, err := newRevision(set, 1, set.Status.CollisionCount) revision, err := newRevision(set, 1, set.Status.CollisionCount)
if err != nil { if err != nil {
@ -241,7 +248,7 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
pods, err := spc.podsLister.Pods(set.Namespace).List(selector) pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -272,13 +279,13 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) {
} }
func TestStatefulSetControllerAddPod(t *testing.T) { func TestStatefulSetControllerAddPod(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
pod1 := newStatefulSetPod(set1, 0) pod1 := newStatefulSetPod(set1, 0)
pod2 := newStatefulSetPod(set2, 0) pod2 := newStatefulSetPod(set2, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
ssc.addPod(pod1) ssc.addPod(pod1)
key, done := ssc.queue.Get() key, done := ssc.queue.Get()
@ -304,7 +311,7 @@ func TestStatefulSetControllerAddPod(t *testing.T) {
} }
func TestStatefulSetControllerAddPodOrphan(t *testing.T) { func TestStatefulSetControllerAddPodOrphan(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
@ -312,9 +319,9 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) {
set3.Name = "foo3" set3.Name = "foo3"
set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"} set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"}
pod := newStatefulSetPod(set1, 0) pod := newStatefulSetPod(set1, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
spc.setsIndexer.Add(set3) om.setsIndexer.Add(set3)
// Make pod an orphan. Expect matching sets to be queued. // Make pod an orphan. Expect matching sets to be queued.
pod.OwnerReferences = nil pod.OwnerReferences = nil
@ -325,7 +332,7 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) {
} }
func TestStatefulSetControllerAddPodNoSet(t *testing.T) { func TestStatefulSetControllerAddPodNoSet(t *testing.T) {
ssc, _, _ := newFakeStatefulSetController() ssc, _, _, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
ssc.addPod(pod) ssc.addPod(pod)
@ -337,14 +344,14 @@ func TestStatefulSetControllerAddPodNoSet(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePod(t *testing.T) { func TestStatefulSetControllerUpdatePod(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod1 := newStatefulSetPod(set1, 0) pod1 := newStatefulSetPod(set1, 0)
pod2 := newStatefulSetPod(set2, 0) pod2 := newStatefulSetPod(set2, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
prev := *pod1 prev := *pod1
fakeResourceVersion(pod1) fakeResourceVersion(pod1)
@ -372,7 +379,7 @@ func TestStatefulSetControllerUpdatePod(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) { func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) {
ssc, _, _ := newFakeStatefulSetController() ssc, _, _, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
prev := *pod prev := *pod
@ -386,10 +393,10 @@ func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) { func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
ssc.updatePod(pod, pod) ssc.updatePod(pod, pod)
ssc.queue.ShutDown() ssc.queue.ShutDown()
key, _ := ssc.queue.Get() key, _ := ssc.queue.Get()
@ -399,14 +406,14 @@ func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
pod.OwnerReferences = nil pod.OwnerReferences = nil
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
clone := *pod clone := *pod
clone.Labels = map[string]string{"foo2": "bar2"} clone.Labels = map[string]string{"foo2": "bar2"}
fakeResourceVersion(&clone) fakeResourceVersion(&clone)
@ -417,14 +424,14 @@ func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) { func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
pod2 := newStatefulSetPod(set2, 0) pod2 := newStatefulSetPod(set2, 0)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
clone := *pod clone := *pod
clone.OwnerReferences = pod2.OwnerReferences clone.OwnerReferences = pod2.OwnerReferences
fakeResourceVersion(&clone) fakeResourceVersion(&clone)
@ -435,13 +442,13 @@ func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) {
} }
func TestStatefulSetControllerUpdatePodRelease(t *testing.T) { func TestStatefulSetControllerUpdatePodRelease(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
clone := *pod clone := *pod
clone.OwnerReferences = nil clone.OwnerReferences = nil
fakeResourceVersion(&clone) fakeResourceVersion(&clone)
@ -452,14 +459,14 @@ func TestStatefulSetControllerUpdatePodRelease(t *testing.T) {
} }
func TestStatefulSetControllerDeletePod(t *testing.T) { func TestStatefulSetControllerDeletePod(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod1 := newStatefulSetPod(set1, 0) pod1 := newStatefulSetPod(set1, 0)
pod2 := newStatefulSetPod(set2, 0) pod2 := newStatefulSetPod(set2, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
ssc.deletePod(pod1) ssc.deletePod(pod1)
key, done := ssc.queue.Get() key, done := ssc.queue.Get()
@ -483,13 +490,13 @@ func TestStatefulSetControllerDeletePod(t *testing.T) {
} }
func TestStatefulSetControllerDeletePodOrphan(t *testing.T) { func TestStatefulSetControllerDeletePodOrphan(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod1 := newStatefulSetPod(set1, 0) pod1 := newStatefulSetPod(set1, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
pod1.OwnerReferences = nil pod1.OwnerReferences = nil
ssc.deletePod(pod1) ssc.deletePod(pod1)
@ -499,10 +506,10 @@ func TestStatefulSetControllerDeletePodOrphan(t *testing.T) {
} }
func TestStatefulSetControllerDeletePodTombstone(t *testing.T) { func TestStatefulSetControllerDeletePodTombstone(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 0) pod := newStatefulSetPod(set, 0)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
tombstoneKey, _ := controller.KeyFunc(pod) tombstoneKey, _ := controller.KeyFunc(pod)
tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod} tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod}
ssc.deletePod(tombstone) ssc.deletePod(tombstone)
@ -517,14 +524,14 @@ func TestStatefulSetControllerDeletePodTombstone(t *testing.T) {
} }
func TestStatefulSetControllerGetStatefulSetsForPod(t *testing.T) { func TestStatefulSetControllerGetStatefulSetsForPod(t *testing.T) {
ssc, spc, _ := newFakeStatefulSetController() ssc, _, om, _ := newFakeStatefulSetController()
set1 := newStatefulSet(3) set1 := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
set2.Name = "foo2" set2.Name = "foo2"
pod := newStatefulSetPod(set1, 0) pod := newStatefulSetPod(set1, 0)
spc.setsIndexer.Add(set1) om.setsIndexer.Add(set1)
spc.setsIndexer.Add(set2) om.setsIndexer.Add(set2)
spc.podsIndexer.Add(pod) om.podsIndexer.Add(pod)
sets := ssc.getStatefulSetsForPod(pod) sets := ssc.getStatefulSetsForPod(pod)
if got, want := len(sets), 2; got != want { if got, want := len(sets), 2; got != want {
t.Errorf("len(sets) = %v, want %v", got, want) t.Errorf("len(sets) = %v, want %v", got, want)
@ -546,12 +553,12 @@ func TestGetPodsForStatefulSetAdopt(t *testing.T) {
pod4.OwnerReferences = nil pod4.OwnerReferences = nil
pod4.Name = "x" + pod4.Name pod4.Name = "x" + pod4.Name
ssc, spc, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4) ssc, _, om, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4)
spc.podsIndexer.Add(pod1) om.podsIndexer.Add(pod1)
spc.podsIndexer.Add(pod2) om.podsIndexer.Add(pod2)
spc.podsIndexer.Add(pod3) om.podsIndexer.Add(pod3)
spc.podsIndexer.Add(pod4) om.podsIndexer.Add(pod4)
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -588,10 +595,10 @@ func TestAdoptOrphanRevisions(t *testing.T) {
ss1Rev2.Namespace = ss1.Namespace ss1Rev2.Namespace = ss1.Namespace
ss1Rev2.OwnerReferences = []metav1.OwnerReference{} ss1Rev2.OwnerReferences = []metav1.OwnerReference{}
ssc, spc, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2) ssc, _, om, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2)
spc.revisionsIndexer.Add(ss1Rev1) om.revisionsIndexer.Add(ss1Rev1)
spc.revisionsIndexer.Add(ss1Rev2) om.revisionsIndexer.Add(ss1Rev2)
err = ssc.adoptOrphanRevisions(context.TODO(), ss1) err = ssc.adoptOrphanRevisions(context.TODO(), ss1)
if err != nil { if err != nil {
@ -615,7 +622,7 @@ func TestAdoptOrphanRevisions(t *testing.T) {
func TestGetPodsForStatefulSetRelease(t *testing.T) { func TestGetPodsForStatefulSetRelease(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
ssc, spc, _ := newFakeStatefulSetController(set) ssc, _, om, _ := newFakeStatefulSetController(set)
pod1 := newStatefulSetPod(set, 1) pod1 := newStatefulSetPod(set, 1)
// pod2 is owned but has wrong name. // pod2 is owned but has wrong name.
pod2 := newStatefulSetPod(set, 2) pod2 := newStatefulSetPod(set, 2)
@ -628,9 +635,10 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) {
pod4.OwnerReferences = nil pod4.OwnerReferences = nil
pod4.Labels = nil pod4.Labels = nil
spc.podsIndexer.Add(pod1) om.podsIndexer.Add(pod1)
spc.podsIndexer.Add(pod2) om.podsIndexer.Add(pod2)
spc.podsIndexer.Add(pod3) om.podsIndexer.Add(pod3)
om.podsIndexer.Add(pod4)
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -651,10 +659,255 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) {
} }
} }
func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *fakeStatefulPodControl, history.Interface) { func TestOrphanedPodsWithPVCDeletePolicy(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
testFn := func(t *testing.T, scaledownPolicy, deletionPolicy apps.PersistentVolumeClaimRetentionPolicyType) {
set := newStatefulSet(4)
*set.Spec.Replicas = 2
set.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = scaledownPolicy
set.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = deletionPolicy
ssc, _, om, _ := newFakeStatefulSetController(set)
om.setsIndexer.Add(set)
pods := []*v1.Pod{}
pods = append(pods, newStatefulSetPod(set, 0))
// pod1 is orphaned
pods = append(pods, newStatefulSetPod(set, 1))
pods[1].OwnerReferences = nil
// pod2 is owned but has wrong name.
pods = append(pods, newStatefulSetPod(set, 2))
pods[2].Name = "x" + pods[2].Name
ssc.kubeClient.(*fake.Clientset).PrependReactor("patch", "pods", func(action core.Action) (bool, runtime.Object, error) {
patch := action.(core.PatchAction).GetPatch()
target := action.(core.PatchAction).GetName()
var pod *v1.Pod
for _, p := range pods {
if p.Name == target {
pod = p
break
}
}
if pod == nil {
t.Fatalf("Can't find patch target %s", target)
}
original, err := json.Marshal(pod)
if err != nil {
t.Fatalf("failed to marshal original pod %s: %v", pod.Name, err)
}
updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Pod{})
if err != nil {
t.Fatalf("failed to apply strategic merge patch %q on node %s: %v", patch, pod.Name, err)
}
if err := json.Unmarshal(updated, pod); err != nil {
t.Fatalf("failed to unmarshal updated pod %s: %v", pod.Name, err)
}
return true, pod, nil
})
for _, pod := range pods {
om.podsIndexer.Add(pod)
claims := getPersistentVolumeClaims(set, pod)
for _, claim := range claims {
om.CreateClaim(&claim)
}
}
for i := range pods {
if _, err := om.setPodReady(set, i); err != nil {
t.Errorf("%d: %v", i, err)
}
if _, err := om.setPodRunning(set, i); err != nil {
t.Errorf("%d: %v", i, err)
}
}
// First sync to manage orphaned pod, then set replicas.
ssc.enqueueStatefulSet(set)
fakeWorker(ssc)
*set.Spec.Replicas = 0 // Put an ownerRef for all scale-down deleted PVCs.
ssc.enqueueStatefulSet(set)
fakeWorker(ssc)
hasNamedOwnerRef := func(claim *v1.PersistentVolumeClaim, name string) bool {
for _, ownerRef := range claim.GetOwnerReferences() {
if ownerRef.Name == name {
return true
}
}
return false
}
verifyOwnerRefs := func(claim *v1.PersistentVolumeClaim, condemned bool) {
podName := getClaimPodName(set, claim)
const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType
const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType
switch {
case scaledownPolicy == retain && deletionPolicy == retain:
if hasNamedOwnerRef(claim, podName) || hasNamedOwnerRef(claim, set.Name) {
t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences())
}
case scaledownPolicy == retain && deletionPolicy == delete:
if hasNamedOwnerRef(claim, podName) || !hasNamedOwnerRef(claim, set.Name) {
t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences())
}
case scaledownPolicy == delete && deletionPolicy == retain:
if hasNamedOwnerRef(claim, podName) != condemned || hasNamedOwnerRef(claim, set.Name) {
t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences())
}
case scaledownPolicy == delete && deletionPolicy == delete:
if hasNamedOwnerRef(claim, podName) != condemned || !hasNamedOwnerRef(claim, set.Name) {
t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences())
}
}
}
claims, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(labels.Everything())
if len(claims) != len(pods) {
t.Errorf("Unexpected number of claims: %d", len(claims))
}
for _, claim := range claims {
// Only the first pod and the reclaimed orphan pod should have owner refs.
switch claim.Name {
case "datadir-foo-0", "datadir-foo-1":
verifyOwnerRefs(claim, false)
case "datadir-foo-2":
if hasNamedOwnerRef(claim, getClaimPodName(set, claim)) || hasNamedOwnerRef(claim, set.Name) {
t.Errorf("unexpected ownerRefs for %s: %v", claim.Name, claim.GetOwnerReferences())
}
default:
t.Errorf("Unexpected claim %s", claim.Name)
}
}
}
policies := []apps.PersistentVolumeClaimRetentionPolicyType{
apps.RetainPersistentVolumeClaimRetentionPolicyType,
apps.DeletePersistentVolumeClaimRetentionPolicyType,
}
for _, scaledownPolicy := range policies {
for _, deletionPolicy := range policies {
testName := fmt.Sprintf("ScaleDown:%s/SetDeletion:%s", scaledownPolicy, deletionPolicy)
t.Run(testName, func(t *testing.T) { testFn(t, scaledownPolicy, deletionPolicy) })
}
}
}
func TestStaleOwnerRefOnScaleup(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
{
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
{
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType,
},
} {
onPolicy := func(msg string, args ...interface{}) string {
return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...)
}
set := newStatefulSet(3)
set.Spec.PersistentVolumeClaimRetentionPolicy = policy
ssc, spc, om, _ := newFakeStatefulSetController(set)
if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf(onPolicy("Failed to turn up StatefulSet : %s", err))
}
var err error
if set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name); err != nil {
t.Errorf(onPolicy("Could not get scaled up set: %v", err))
}
if set.Status.Replicas != 3 {
t.Errorf(onPolicy("set.Status.Replicas = %v; want 3", set.Status.Replicas))
}
*set.Spec.Replicas = 2
if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf(onPolicy("Failed to scale down StatefulSet : msg, %s", err))
}
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err))
}
if set.Status.Replicas != 2 {
t.Errorf(onPolicy("Failed to scale statefulset to 2 replicas"))
}
var claim *v1.PersistentVolumeClaim
claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2")
if err != nil {
t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2"))
}
refs := claim.GetOwnerReferences()
if len(refs) != 1 {
t.Errorf(onPolicy("Expected only one refs: %v", refs))
}
// Make the pod ref stale.
for i := range refs {
if refs[i].Name == "foo-2" {
refs[i].UID = "stale"
break
}
}
claim.SetOwnerReferences(refs)
if err = om.claimsIndexer.Update(claim); err != nil {
t.Errorf(onPolicy("Could not update claim with new owner ref: %v", err))
}
*set.Spec.Replicas = 3
// Until the stale PVC goes away, the scale up should never finish. Run 10 iterations, then delete the PVC.
if err := scaleUpStatefulSetControllerBounded(set, ssc, spc, om, 10); err != nil {
t.Errorf(onPolicy("Failed attempt to scale StatefulSet back up: %v", err))
}
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err))
}
if set.Status.Replicas != 2 {
t.Errorf(onPolicy("Expected set to stay at two replicas"))
}
claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2")
if err != nil {
t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2"))
}
refs = claim.GetOwnerReferences()
if len(refs) != 1 {
t.Errorf(onPolicy("Unexpected change to condemned pvc ownerRefs: %v", refs))
}
foundPodRef := false
for i := range refs {
if refs[i].UID == "stale" {
foundPodRef = true
break
}
}
if !foundPodRef {
t.Errorf(onPolicy("Claim ref unexpectedly changed: %v", refs))
}
if err = om.claimsIndexer.Delete(claim); err != nil {
t.Errorf(onPolicy("Could not delete stale pvc: %v", err))
}
if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil {
t.Errorf(onPolicy("Failed to scale StatefulSet back up: %v", err))
}
set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name)
if err != nil {
t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err))
}
if set.Status.Replicas != 3 {
t.Errorf(onPolicy("Failed to scale set back up once PVC was deleted"))
}
}
}
func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *StatefulPodControl, *fakeObjectManager, history.Interface) {
client := fake.NewSimpleClientset(initialObjects...) client := fake.NewSimpleClientset(initialObjects...)
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
fpc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) om := newFakeObjectManager(informerFactory)
spc := NewStatefulPodControlFromManager(om, &noopRecorder{})
ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets())
ssc := NewStatefulSetController( ssc := NewStatefulSetController(
informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Pods(),
@ -667,9 +920,9 @@ func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSe
ssc.podListerSynced = alwaysReady ssc.podListerSynced = alwaysReady
ssc.setListerSynced = alwaysReady ssc.setListerSynced = alwaysReady
recorder := record.NewFakeRecorder(10) recorder := record.NewFakeRecorder(10)
ssc.control = NewDefaultStatefulSetControl(fpc, ssu, ssh, recorder) ssc.control = NewDefaultStatefulSetControl(spc, ssu, ssh, recorder)
return ssc, fpc, ssh return ssc, spc, om, ssh
} }
func fakeWorker(ssc *StatefulSetController) { func fakeWorker(ssc *StatefulSetController) {
@ -687,21 +940,27 @@ func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod {
return pods[ordinal] return pods[ordinal]
} }
func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error {
spc.setsIndexer.Add(set) return scaleUpStatefulSetControllerBounded(set, ssc, spc, om, -1)
}
func scaleUpStatefulSetControllerBounded(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager, maxIterations int) error {
om.setsIndexer.Add(set)
ssc.enqueueStatefulSet(set) ssc.enqueueStatefulSet(set)
fakeWorker(ssc) fakeWorker(ssc)
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
if err != nil { if err != nil {
return err return err
} }
for set.Status.ReadyReplicas < *set.Spec.Replicas { iterations := 0
pods, err := spc.podsLister.Pods(set.Namespace).List(selector) for (maxIterations < 0 || iterations < maxIterations) && set.Status.ReadyReplicas < *set.Spec.Replicas {
iterations++
pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
return err return err
} }
ord := len(pods) - 1 ord := len(pods) - 1
if pods, err = spc.setPodPending(set, ord); err != nil { if pods, err = om.setPodPending(set, ord); err != nil {
return err return err
} }
pod := getPodAtOrdinal(pods, ord) pod := getPodAtOrdinal(pods, ord)
@ -709,7 +968,7 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl
fakeWorker(ssc) fakeWorker(ssc)
pod = getPodAtOrdinal(pods, ord) pod = getPodAtOrdinal(pods, ord)
prev := *pod prev := *pod
if pods, err = spc.setPodRunning(set, ord); err != nil { if pods, err = om.setPodRunning(set, ord); err != nil {
return err return err
} }
pod = getPodAtOrdinal(pods, ord) pod = getPodAtOrdinal(pods, ord)
@ -717,31 +976,31 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl
fakeWorker(ssc) fakeWorker(ssc)
pod = getPodAtOrdinal(pods, ord) pod = getPodAtOrdinal(pods, ord)
prev = *pod prev = *pod
if pods, err = spc.setPodReady(set, ord); err != nil { if pods, err = om.setPodReady(set, ord); err != nil {
return err return err
} }
pod = getPodAtOrdinal(pods, ord) pod = getPodAtOrdinal(pods, ord)
ssc.updatePod(&prev, pod) ssc.updatePod(&prev, pod)
fakeWorker(ssc) fakeWorker(ssc)
if err := assertMonotonicInvariants(set, spc); err != nil { if err := assertMonotonicInvariants(set, om); err != nil {
return err return err
} }
obj, _, err := spc.setsIndexer.Get(set) obj, _, err := om.setsIndexer.Get(set)
if err != nil { if err != nil {
return err return err
} }
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
} }
return assertMonotonicInvariants(set, spc) return assertMonotonicInvariants(set, om)
} }
func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error {
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
if err != nil { if err != nil {
return err return err
} }
pods, err := spc.podsLister.Pods(set.Namespace).List(selector) pods, err := om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
return err return err
} }
@ -749,10 +1008,10 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr
pod := getPodAtOrdinal(pods, ord) pod := getPodAtOrdinal(pods, ord)
prev := *pod prev := *pod
fakeResourceVersion(set) fakeResourceVersion(set)
spc.setsIndexer.Add(set) om.setsIndexer.Add(set)
ssc.enqueueStatefulSet(set) ssc.enqueueStatefulSet(set)
fakeWorker(ssc) fakeWorker(ssc)
pods, err = spc.addTerminatingPod(set, ord) pods, err = om.addTerminatingPod(set, ord)
if err != nil { if err != nil {
return err return err
} }
@ -763,13 +1022,13 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr
ssc.deletePod(pod) ssc.deletePod(pod)
fakeWorker(ssc) fakeWorker(ssc)
for set.Status.Replicas > *set.Spec.Replicas { for set.Status.Replicas > *set.Spec.Replicas {
pods, err = spc.podsLister.Pods(set.Namespace).List(selector) pods, err = om.podsLister.Pods(set.Namespace).List(selector)
if err != nil { if err != nil {
return err return err
} }
ord := len(pods) ord := len(pods)
pods, err = spc.addTerminatingPod(set, ord) pods, err = om.addTerminatingPod(set, ord)
if err != nil { if err != nil {
return err return err
} }
@ -779,14 +1038,14 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr
spc.DeleteStatefulPod(set, pod) spc.DeleteStatefulPod(set, pod)
ssc.deletePod(pod) ssc.deletePod(pod)
fakeWorker(ssc) fakeWorker(ssc)
obj, _, err := spc.setsIndexer.Get(set) obj, _, err := om.setsIndexer.Get(set)
if err != nil { if err != nil {
return err return err
} }
set = obj.(*apps.StatefulSet) set = obj.(*apps.StatefulSet)
} }
return assertMonotonicInvariants(set, spc) return assertMonotonicInvariants(set, om)
} }
func rawTemplate(template *v1.PodTemplateSpec) runtime.RawExtension { func rawTemplate(template *v1.PodTemplateSpec) runtime.RawExtension {

View File

@ -29,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog/v2"
podutil "k8s.io/kubernetes/pkg/api/v1/pod" podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/history" "k8s.io/kubernetes/pkg/controller/history"
@ -132,6 +133,179 @@ func storageMatches(set *apps.StatefulSet, pod *v1.Pod) bool {
return true return true
} }
// getPersistentVolumeClaimPolicy returns the PVC policy for a StatefulSet, returning a retain policy if the set policy is nil.
func getPersistentVolumeClaimRetentionPolicy(set *apps.StatefulSet) apps.StatefulSetPersistentVolumeClaimRetentionPolicy {
policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
if set.Spec.PersistentVolumeClaimRetentionPolicy != nil {
policy = *set.Spec.PersistentVolumeClaimRetentionPolicy
}
return policy
}
// claimOwnerMatchesSetAndPod returns false if the ownerRefs of the claim are not set consistently with the
// PVC deletion policy for the StatefulSet.
func claimOwnerMatchesSetAndPod(claim *v1.PersistentVolumeClaim, set *apps.StatefulSet, pod *v1.Pod) bool {
policy := set.Spec.PersistentVolumeClaimRetentionPolicy
const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType
const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType
switch {
default:
klog.Errorf("Unknown policy %v; treating as Retain", set.Spec.PersistentVolumeClaimRetentionPolicy)
fallthrough
case policy.WhenScaled == retain && policy.WhenDeleted == retain:
if hasOwnerRef(claim, set) ||
hasOwnerRef(claim, pod) {
return false
}
case policy.WhenScaled == retain && policy.WhenDeleted == delete:
if !hasOwnerRef(claim, set) ||
hasOwnerRef(claim, pod) {
return false
}
case policy.WhenScaled == delete && policy.WhenDeleted == retain:
if hasOwnerRef(claim, set) {
return false
}
podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas)
if podScaledDown != hasOwnerRef(claim, pod) {
return false
}
case policy.WhenScaled == delete && policy.WhenDeleted == delete:
podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas)
// If a pod is scaled down, there should be no set ref and a pod ref;
// if the pod is not scaled down it's the other way around.
if podScaledDown == hasOwnerRef(claim, set) {
return false
}
if podScaledDown != hasOwnerRef(claim, pod) {
return false
}
}
return true
}
// updateClaimOwnerRefForSetAndPod updates the ownerRefs for the claim according to the deletion policy of
// the StatefulSet. Returns true if the claim was changed and should be updated and false otherwise.
func updateClaimOwnerRefForSetAndPod(claim *v1.PersistentVolumeClaim, set *apps.StatefulSet, pod *v1.Pod) bool {
needsUpdate := false
// Sometimes the version and kind are not set {pod,set}.TypeMeta. These are necessary for the ownerRef.
// This is the case both in real clusters and the unittests.
// TODO: there must be a better way to do this other than hardcoding the pod version?
updateMeta := func(tm *metav1.TypeMeta, kind string) {
if tm.APIVersion == "" {
if kind == "StatefulSet" {
tm.APIVersion = "apps/v1"
} else {
tm.APIVersion = "v1"
}
}
if tm.Kind == "" {
tm.Kind = kind
}
}
podMeta := pod.TypeMeta
updateMeta(&podMeta, "Pod")
setMeta := set.TypeMeta
updateMeta(&setMeta, "StatefulSet")
policy := set.Spec.PersistentVolumeClaimRetentionPolicy
const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType
const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType
switch {
default:
klog.Errorf("Unknown policy %v, treating as Retain", set.Spec.PersistentVolumeClaimRetentionPolicy)
fallthrough
case policy.WhenScaled == retain && policy.WhenDeleted == retain:
needsUpdate = removeOwnerRef(claim, set) || needsUpdate
needsUpdate = removeOwnerRef(claim, pod) || needsUpdate
case policy.WhenScaled == retain && policy.WhenDeleted == delete:
needsUpdate = setOwnerRef(claim, set, &setMeta) || needsUpdate
needsUpdate = removeOwnerRef(claim, pod) || needsUpdate
case policy.WhenScaled == delete && policy.WhenDeleted == retain:
needsUpdate = removeOwnerRef(claim, set) || needsUpdate
podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas)
if podScaledDown {
needsUpdate = setOwnerRef(claim, pod, &podMeta) || needsUpdate
}
if !podScaledDown {
needsUpdate = removeOwnerRef(claim, pod) || needsUpdate
}
case policy.WhenScaled == delete && policy.WhenDeleted == delete:
podScaledDown := getOrdinal(pod) >= int(*set.Spec.Replicas)
if podScaledDown {
needsUpdate = removeOwnerRef(claim, set) || needsUpdate
needsUpdate = setOwnerRef(claim, pod, &podMeta) || needsUpdate
}
if !podScaledDown {
needsUpdate = setOwnerRef(claim, set, &setMeta) || needsUpdate
needsUpdate = removeOwnerRef(claim, pod) || needsUpdate
}
}
return needsUpdate
}
// hasOwnerRef returns true if target has an ownerRef to owner.
func hasOwnerRef(target, owner metav1.Object) bool {
ownerUID := owner.GetUID()
for _, ownerRef := range target.GetOwnerReferences() {
if ownerRef.UID == ownerUID {
return true
}
}
return false
}
// hasStaleOwnerRef returns true if target has a ref to owner that appears to be stale.
func hasStaleOwnerRef(target, owner metav1.Object) bool {
for _, ownerRef := range target.GetOwnerReferences() {
if ownerRef.Name == owner.GetName() && ownerRef.UID != owner.GetUID() {
return true
}
}
return false
}
// setOwnerRef adds owner to the ownerRefs of target, if necessary. Returns true if target needs to be
// updated and false otherwise.
func setOwnerRef(target, owner metav1.Object, ownerType *metav1.TypeMeta) bool {
if hasOwnerRef(target, owner) {
return false
}
ownerRefs := append(
target.GetOwnerReferences(),
metav1.OwnerReference{
APIVersion: ownerType.APIVersion,
Kind: ownerType.Kind,
Name: owner.GetName(),
UID: owner.GetUID(),
})
target.SetOwnerReferences(ownerRefs)
return true
}
// removeOwnerRef removes owner from the ownerRefs of target, if necessary. Returns true if target needs
// to be updated and false otherwise.
func removeOwnerRef(target, owner metav1.Object) bool {
if !hasOwnerRef(target, owner) {
return false
}
ownerUID := owner.GetUID()
oldRefs := target.GetOwnerReferences()
newRefs := make([]metav1.OwnerReference, len(oldRefs)-1)
skip := 0
for i := range oldRefs {
if oldRefs[i].UID == ownerUID {
skip = -1
} else {
newRefs[i+skip] = oldRefs[i]
}
}
target.SetOwnerReferences(newRefs)
return true
}
// getPersistentVolumeClaims gets a map of PersistentVolumeClaims to their template names, as defined in set. The // getPersistentVolumeClaims gets a map of PersistentVolumeClaims to their template names, as defined in set. The
// returned PersistentVolumeClaims are each constructed with a the name specific to the Pod. This name is determined // returned PersistentVolumeClaims are each constructed with a the name specific to the Pod. This name is determined
// by getPersistentVolumeClaimName. // by getPersistentVolumeClaimName.
@ -278,7 +452,10 @@ func newVersionedStatefulSetPod(currentSet, updateSet *apps.StatefulSet, current
// Match check if the given StatefulSet's template matches the template stored in the given history. // Match check if the given StatefulSet's template matches the template stored in the given history.
func Match(ss *apps.StatefulSet, history *apps.ControllerRevision) (bool, error) { func Match(ss *apps.StatefulSet, history *apps.ControllerRevision) (bool, error) {
patch, err := getPatch(ss) // Encoding the set for the patch may update its GVK metadata, which causes data races if this
// set is in an informer cache.
clone := ss.DeepCopy()
patch, err := getPatch(clone)
if err != nil { if err != nil {
return false, err return false, err
} }

View File

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"reflect" "reflect"
"regexp"
"sort" "sort"
"strconv" "strconv"
"testing" "testing"
@ -27,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
apps "k8s.io/api/apps/v1" apps "k8s.io/api/apps/v1"
@ -35,6 +37,28 @@ import (
"k8s.io/kubernetes/pkg/controller/history" "k8s.io/kubernetes/pkg/controller/history"
) )
// noopRecorder is an EventRecorder that does nothing. record.FakeRecorder has a fixed
// buffer size, which causes tests to hang if that buffer's exceeded.
type noopRecorder struct{}
func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {}
func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
}
func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) {
}
// getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching.
func getClaimPodName(set *apps.StatefulSet, claim *v1.PersistentVolumeClaim) string {
podName := ""
statefulClaimRegex := regexp.MustCompile(fmt.Sprintf(".*-(%s-[0-9]+)$", set.Name))
matches := statefulClaimRegex.FindStringSubmatch(claim.Name)
if len(matches) != 2 {
return podName
}
return matches[1]
}
func TestGetParentNameAndOrdinal(t *testing.T) { func TestGetParentNameAndOrdinal(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 1) pod := newStatefulSetPod(set, 1)
@ -51,6 +75,28 @@ func TestGetParentNameAndOrdinal(t *testing.T) {
} }
} }
func TestGetClaimPodName(t *testing.T) {
set := apps.StatefulSet{}
set.Name = "my-set"
claim := v1.PersistentVolumeClaim{}
claim.Name = "volume-my-set-2"
if pod := getClaimPodName(&set, &claim); pod != "my-set-2" {
t.Errorf("Expected my-set-2 found %s", pod)
}
claim.Name = "long-volume-my-set-20"
if pod := getClaimPodName(&set, &claim); pod != "my-set-20" {
t.Errorf("Expected my-set-20 found %s", pod)
}
claim.Name = "volume-2-my-set"
if pod := getClaimPodName(&set, &claim); pod != "" {
t.Errorf("Expected empty string found %s", pod)
}
claim.Name = "volume-pod-2"
if pod := getClaimPodName(&set, &claim); pod != "" {
t.Errorf("Expected empty string found %s", pod)
}
}
func TestIsMemberOf(t *testing.T) { func TestIsMemberOf(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
set2 := newStatefulSet(3) set2 := newStatefulSet(3)
@ -180,6 +226,350 @@ func TestUpdateStorage(t *testing.T) {
} }
} }
func TestGetPersistentVolumeClaimRetentionPolicy(t *testing.T) {
retainPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
scaledownPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
}
set := apps.StatefulSet{}
set.Spec.PersistentVolumeClaimRetentionPolicy = &retainPolicy
got := getPersistentVolumeClaimRetentionPolicy(&set)
if got.WhenScaled != apps.RetainPersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType {
t.Errorf("Expected retain policy")
}
set.Spec.PersistentVolumeClaimRetentionPolicy = &scaledownPolicy
got = getPersistentVolumeClaimRetentionPolicy(&set)
if got.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType {
t.Errorf("Expected scaledown policy")
}
}
func TestClaimOwnerMatchesSetAndPod(t *testing.T) {
testCases := []struct {
name string
scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType
setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType
needsPodRef bool
needsSetRef bool
replicas int32
ordinal int
}{
{
name: "retain",
scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
needsPodRef: false,
needsSetRef: false,
},
{
name: "on SS delete",
scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
needsPodRef: false,
needsSetRef: true,
},
{
name: "on scaledown only, condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
needsPodRef: true,
needsSetRef: false,
replicas: 2,
ordinal: 2,
},
{
name: "on scaledown only, remains",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
needsPodRef: false,
needsSetRef: false,
replicas: 2,
ordinal: 1,
},
{
name: "on both, condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
needsPodRef: true,
needsSetRef: false,
replicas: 2,
ordinal: 2,
},
{
name: "on both, remains",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
needsPodRef: false,
needsSetRef: true,
replicas: 2,
ordinal: 1,
},
}
for _, tc := range testCases {
for _, useOtherRefs := range []bool{false, true} {
for _, setPodRef := range []bool{false, true} {
for _, setSetRef := range []bool{false, true} {
claim := v1.PersistentVolumeClaim{}
claim.Name = "target-claim"
pod := v1.Pod{}
pod.Name = fmt.Sprintf("pod-%d", tc.ordinal)
pod.GetObjectMeta().SetUID("pod-123")
set := apps.StatefulSet{}
set.Name = "stateful-set"
set.GetObjectMeta().SetUID("ss-456")
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: tc.scaleDownPolicy,
WhenDeleted: tc.setDeletePolicy,
}
set.Spec.Replicas = &tc.replicas
if setPodRef {
setOwnerRef(&claim, &pod, &pod.TypeMeta)
}
if setSetRef {
setOwnerRef(&claim, &set, &set.TypeMeta)
}
if useOtherRefs {
randomObject1 := v1.Pod{}
randomObject1.Name = "rand1"
randomObject1.GetObjectMeta().SetUID("rand1-abc")
randomObject2 := v1.Pod{}
randomObject2.Name = "rand2"
randomObject2.GetObjectMeta().SetUID("rand2-def")
setOwnerRef(&claim, &randomObject1, &randomObject1.TypeMeta)
setOwnerRef(&claim, &randomObject2, &randomObject2.TypeMeta)
}
shouldMatch := setPodRef == tc.needsPodRef && setSetRef == tc.needsSetRef
if claimOwnerMatchesSetAndPod(&claim, &set, &pod) != shouldMatch {
t.Errorf("Bad match for %s with pod=%v,set=%v,others=%v", tc.name, setPodRef, setSetRef, useOtherRefs)
}
}
}
}
}
}
func TestUpdateClaimOwnerRefForSetAndPod(t *testing.T) {
testCases := []struct {
name string
scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType
setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType
condemned bool
needsPodRef bool
needsSetRef bool
}{
{
name: "retain",
scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
condemned: false,
needsPodRef: false,
needsSetRef: false,
},
{
name: "delete with set",
scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
condemned: false,
needsPodRef: false,
needsSetRef: true,
},
{
name: "delete with scaledown, not condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
condemned: false,
needsPodRef: false,
needsSetRef: false,
},
{
name: "delete on scaledown, condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType,
condemned: true,
needsPodRef: true,
needsSetRef: false,
},
{
name: "delete on both, not condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
condemned: false,
needsPodRef: false,
needsSetRef: true,
},
{
name: "delete on both, condemned",
scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType,
condemned: true,
needsPodRef: true,
needsSetRef: false,
},
}
for _, tc := range testCases {
for _, hasPodRef := range []bool{true, false} {
for _, hasSetRef := range []bool{true, false} {
set := apps.StatefulSet{}
set.Name = "ss"
numReplicas := int32(5)
set.Spec.Replicas = &numReplicas
set.SetUID("ss-123")
set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: tc.scaleDownPolicy,
WhenDeleted: tc.setDeletePolicy,
}
pod := v1.Pod{}
if tc.condemned {
pod.Name = "pod-8"
} else {
pod.Name = "pod-1"
}
pod.SetUID("pod-456")
claim := v1.PersistentVolumeClaim{}
if hasPodRef {
setOwnerRef(&claim, &pod, &pod.TypeMeta)
}
if hasSetRef {
setOwnerRef(&claim, &set, &set.TypeMeta)
}
needsUpdate := hasPodRef != tc.needsPodRef || hasSetRef != tc.needsSetRef
shouldUpdate := updateClaimOwnerRefForSetAndPod(&claim, &set, &pod)
if shouldUpdate != needsUpdate {
t.Errorf("Bad update for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef)
}
if hasOwnerRef(&claim, &pod) != tc.needsPodRef {
t.Errorf("Bad pod ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef)
}
if hasOwnerRef(&claim, &set) != tc.needsSetRef {
t.Errorf("Bad set ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef)
}
}
}
}
}
func TestHasOwnerRef(t *testing.T) {
target := v1.Pod{}
target.SetOwnerReferences([]metav1.OwnerReference{
{UID: "123"}, {UID: "456"}})
ownerA := v1.Pod{}
ownerA.GetObjectMeta().SetUID("123")
ownerB := v1.Pod{}
ownerB.GetObjectMeta().SetUID("789")
if !hasOwnerRef(&target, &ownerA) {
t.Error("Missing owner")
}
if hasOwnerRef(&target, &ownerB) {
t.Error("Unexpected owner")
}
}
func TestHasStaleOwnerRef(t *testing.T) {
target := v1.Pod{}
target.SetOwnerReferences([]metav1.OwnerReference{
{Name: "bob", UID: "123"}, {Name: "shirley", UID: "456"}})
ownerA := v1.Pod{}
ownerA.SetUID("123")
ownerA.Name = "bob"
ownerB := v1.Pod{}
ownerB.Name = "shirley"
ownerB.SetUID("789")
ownerC := v1.Pod{}
ownerC.Name = "yvonne"
ownerC.SetUID("345")
if hasStaleOwnerRef(&target, &ownerA) {
t.Error("ownerA should not be stale")
}
if !hasStaleOwnerRef(&target, &ownerB) {
t.Error("ownerB should be stale")
}
if hasStaleOwnerRef(&target, &ownerC) {
t.Error("ownerC should not be stale")
}
}
func TestSetOwnerRef(t *testing.T) {
target := v1.Pod{}
ownerA := v1.Pod{}
ownerA.Name = "A"
ownerA.GetObjectMeta().SetUID("ABC")
if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != true {
t.Errorf("Unexpected lack of update")
}
ownerRefs := target.GetObjectMeta().GetOwnerReferences()
if len(ownerRefs) != 1 {
t.Errorf("Unexpected owner ref count: %d", len(ownerRefs))
}
if ownerRefs[0].UID != "ABC" {
t.Errorf("Unexpected owner UID %v", ownerRefs[0].UID)
}
if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != false {
t.Errorf("Unexpected update")
}
if len(target.GetObjectMeta().GetOwnerReferences()) != 1 {
t.Error("Unexpected duplicate reference")
}
ownerB := v1.Pod{}
ownerB.Name = "B"
ownerB.GetObjectMeta().SetUID("BCD")
if setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) != true {
t.Error("Unexpected lack of second update")
}
ownerRefs = target.GetObjectMeta().GetOwnerReferences()
if len(ownerRefs) != 2 {
t.Errorf("Unexpected owner ref count: %d", len(ownerRefs))
}
if ownerRefs[0].UID != "ABC" || ownerRefs[1].UID != "BCD" {
t.Errorf("Bad second ownerRefs: %v", ownerRefs)
}
}
func TestRemoveOwnerRef(t *testing.T) {
target := v1.Pod{}
ownerA := v1.Pod{}
ownerA.Name = "A"
ownerA.GetObjectMeta().SetUID("ABC")
if removeOwnerRef(&target, &ownerA) != false {
t.Error("Unexpected update on empty remove")
}
setOwnerRef(&target, &ownerA, &ownerA.TypeMeta)
if removeOwnerRef(&target, &ownerA) != true {
t.Error("Unexpected lack of update")
}
if len(target.GetObjectMeta().GetOwnerReferences()) != 0 {
t.Error("Unexpected owner reference remains")
}
ownerB := v1.Pod{}
ownerB.Name = "B"
ownerB.GetObjectMeta().SetUID("BCD")
setOwnerRef(&target, &ownerA, &ownerA.TypeMeta)
if removeOwnerRef(&target, &ownerB) != false {
t.Error("Unexpected update for mismatched owner")
}
if len(target.GetObjectMeta().GetOwnerReferences()) != 1 {
t.Error("Missing ref after no-op remove")
}
setOwnerRef(&target, &ownerB, &ownerB.TypeMeta)
if removeOwnerRef(&target, &ownerA) != true {
t.Error("Missing update for second remove")
}
ownerRefs := target.GetObjectMeta().GetOwnerReferences()
if len(ownerRefs) != 1 {
t.Error("Extra ref after second remove")
}
if ownerRefs[0].UID != "BCD" {
t.Error("Bad UID after second remove")
}
}
func TestIsRunningAndReady(t *testing.T) { func TestIsRunningAndReady(t *testing.T) {
set := newStatefulSet(3) set := newStatefulSet(3)
pod := newStatefulSetPod(set, 1) pod := newStatefulSetPod(set, 1)
@ -387,7 +777,8 @@ func newPod() *v1.Pod {
func newPVC(name string) v1.PersistentVolumeClaim { func newPVC(name string) v1.PersistentVolumeClaim {
return v1.PersistentVolumeClaim{ return v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Name: name, Namespace: "default",
Name: name,
}, },
Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{ Resources: v1.ResourceRequirements{
@ -452,6 +843,10 @@ func newStatefulSetWithVolumes(replicas int, name string, petMounts []v1.VolumeM
VolumeClaimTemplates: claims, VolumeClaimTemplates: claims,
ServiceName: "governingsvc", ServiceName: "governingsvc",
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
RevisionHistoryLimit: func() *int32 { RevisionHistoryLimit: func() *int32 {
limit := int32(2) limit := int32(2)
return &limit return &limit
@ -499,6 +894,10 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m
MatchExpressions: testMatchExpressions, MatchExpressions: testMatchExpressions,
}, },
Replicas: func() *int32 { i := int32(replicas); return &i }(), Replicas: func() *int32 { i := int32(replicas); return &i }(),
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
},
Template: v1.PodTemplateSpec{ Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Labels: labels, Labels: labels,
@ -525,7 +924,7 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m
}, },
VolumeClaimTemplates: []v1.PersistentVolumeClaim{ VolumeClaimTemplates: []v1.PersistentVolumeClaim{
{ {
ObjectMeta: metav1.ObjectMeta{Name: "datadir"}, ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "datadir"},
Spec: v1.PersistentVolumeClaimSpec{ Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{ Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{ Requests: v1.ResourceList{

View File

@ -610,6 +610,12 @@ const (
// alpha: v1.21 // alpha: v1.21
VolumeCapacityPriority featuregate.Feature = "VolumeCapacityPriority" VolumeCapacityPriority featuregate.Feature = "VolumeCapacityPriority"
// owner: @mattcary
// alpha: v1.22
//
// Enables policies controlling deletion of PVCs created by a StatefulSet.
StatefulSetAutoDeletePVC featuregate.Feature = "StatefulSetAutoDeletePVC"
// owner: @ahg-g // owner: @ahg-g
// alpha: v1.21 // alpha: v1.21
// beta: v1.22 // beta: v1.22
@ -941,6 +947,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
ProbeTerminationGracePeriod: {Default: false, PreRelease: featuregate.Beta}, // Default to false in beta 1.22, set to true in 1.24 ProbeTerminationGracePeriod: {Default: false, PreRelease: featuregate.Beta}, // Default to false in beta 1.22, set to true in 1.24
NodeSwap: {Default: false, PreRelease: featuregate.Alpha}, NodeSwap: {Default: false, PreRelease: featuregate.Alpha},
PodDeletionCost: {Default: true, PreRelease: featuregate.Beta}, PodDeletionCost: {Default: true, PreRelease: featuregate.Beta},
StatefulSetAutoDeletePVC: {Default: false, PreRelease: featuregate.Alpha},
TopologyAwareHints: {Default: false, PreRelease: featuregate.Beta}, TopologyAwareHints: {Default: false, PreRelease: featuregate.Beta},
PodAffinityNamespaceSelector: {Default: true, PreRelease: featuregate.Beta}, PodAffinityNamespaceSelector: {Default: true, PreRelease: featuregate.Beta},
ServiceLoadBalancerClass: {Default: true, PreRelease: featuregate.Beta}, ServiceLoadBalancerClass: {Default: true, PreRelease: featuregate.Beta},

View File

@ -18,18 +18,18 @@ package statefulset
import ( import (
"context" "context"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
apiequality "k8s.io/apimachinery/pkg/api/equality" apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/storage/names" "k8s.io/apiserver/pkg/storage/names"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/api/pod" "k8s.io/kubernetes/pkg/api/pod"
"k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/apps"
"k8s.io/kubernetes/pkg/apis/apps/validation" "k8s.io/kubernetes/pkg/apis/apps/validation"
"k8s.io/kubernetes/pkg/features"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath" "sigs.k8s.io/structured-merge-diff/v4/fieldpath"
) )
@ -74,6 +74,7 @@ func (statefulSetStrategy) PrepareForCreate(ctx context.Context, obj runtime.Obj
statefulSet.Status = apps.StatefulSetStatus{} statefulSet.Status = apps.StatefulSetStatus{}
statefulSet.Generation = 1 statefulSet.Generation = 1
dropStatefulSetDisabledFields(statefulSet, nil) dropStatefulSetDisabledFields(statefulSet, nil)
pod.DropDisabledTemplateFields(&statefulSet.Spec.Template, nil) pod.DropDisabledTemplateFields(&statefulSet.Spec.Template, nil)
} }
@ -108,6 +109,12 @@ func dropStatefulSetDisabledFields(newSS *apps.StatefulSet, oldSS *apps.Stateful
newSS.Spec.MinReadySeconds = int32(0) newSS.Spec.MinReadySeconds = int32(0)
} }
} }
if !utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
if oldSS == nil || oldSS.Spec.PersistentVolumeClaimRetentionPolicy == nil {
newSS.Spec.PersistentVolumeClaimRetentionPolicy = nil
}
}
} }
// minReadySecondsFieldsInUse returns true if fields related to StatefulSet minReadySeconds are set and // minReadySecondsFieldsInUse returns true if fields related to StatefulSet minReadySeconds are set and

View File

@ -178,13 +178,106 @@ func TestStatefulSetStrategy(t *testing.T) {
if len(errs) == 0 { if len(errs) == 0 {
t.Errorf("updating only spec.Replicas is allowed on a statefulset: %v", errs) t.Errorf("updating only spec.Replicas is allowed on a statefulset: %v", errs)
} }
expectedUpdateErrorString := "spec: Forbidden: updates to statefulset spec for fields other than 'replicas'," + expectedUpdateErrorString := "spec: Forbidden: updates to statefulset spec for fields other than" +
" 'template', 'updateStrategy' and 'minReadySeconds' are forbidden" " 'replicas', 'template', 'updateStrategy' and 'persistentVolumeClaimRetentionPolicy' are forbidden"
if errs[0].Error() != expectedUpdateErrorString { if errs[0].Error() != expectedUpdateErrorString {
t.Errorf("expected error string %v", errs[0].Error()) t.Errorf("expected error string %v", errs[0].Error())
} }
}) })
validPs = &apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: ps.Name, Namespace: ps.Namespace, ResourceVersion: "1", Generation: 1},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: ps.Spec.Selector,
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
Status: apps.StatefulSetStatus{Replicas: 4},
}
t.Run("when StatefulSetAutoDeletePVC feature gate is enabled, PersistentVolumeClaimRetentionPolicy should be updated", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
// Test creation
ps := &apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: ps.Spec.Selector,
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.PersistentVolumeClaimRetentionPolicyType("invalid policy"),
},
},
}
Strategy.PrepareForCreate(ctx, ps)
errs := Strategy.Validate(ctx, ps)
if len(errs) == 0 {
t.Errorf("expected failure when PersistentVolumeClaimRetentionPolicy is invalid")
}
expectedCreateErrorString := "spec.persistentVolumeClaimRetentionPolicy.whenDeleted: Unsupported value: \"invalid policy\": supported values: \"Retain\", \"Delete\""
if errs[0].Error() != expectedCreateErrorString {
t.Errorf("mismatched error string %v (expected %v)", errs[0].Error(), expectedCreateErrorString)
}
Strategy.PrepareForUpdate(ctx, validPs, ps)
errs = Strategy.ValidateUpdate(ctx, validPs, ps)
if len(errs) != 0 {
t.Errorf("updates to PersistentVolumeClaimRetentionPolicy should be allowed: %v", errs)
}
invalidPs := ps
invalidPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType("invalid type")
Strategy.PrepareForUpdate(ctx, validPs, invalidPs)
errs = Strategy.ValidateUpdate(ctx, validPs, ps)
if len(errs) != 0 {
t.Errorf("invalid updates to PersistentVolumeClaimRetentionPolicyType should be allowed: %v", errs)
}
if validPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType || validPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType {
t.Errorf("expected PersistentVolumeClaimRetentionPolicy to be updated: %v", errs)
}
})
t.Run("when StatefulSetAutoDeletePVC feature gate is disabled, PersistentVolumeClaimRetentionPolicy should not be updated", func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)()
// Test creation
ps := &apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: apps.StatefulSetSpec{
PodManagementPolicy: apps.OrderedReadyPodManagement,
Selector: ps.Spec.Selector,
Template: validPodTemplate.Template,
UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType,
WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType,
},
},
}
Strategy.PrepareForCreate(ctx, ps)
errs := Strategy.Validate(ctx, ps)
if len(errs) != 0 {
t.Errorf("unexpected failure with PersistentVolumeClaimRetentionPolicy: %v", errs)
}
if ps.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType || ps.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType {
t.Errorf("expected invalid PersistentVolumeClaimRetentionPolicy to be defaulted to Retain, but got %v", ps.Spec.PersistentVolumeClaimRetentionPolicy)
}
Strategy.PrepareForUpdate(ctx, validPs, ps)
errs = Strategy.ValidateUpdate(ctx, validPs, ps)
if len(errs) != 0 {
t.Errorf("updates to PersistentVolumeClaimRetentionPolicy should be allowed: %v", errs)
}
invalidPs := ps
invalidPs.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = apps.PersistentVolumeClaimRetentionPolicyType("invalid type")
Strategy.PrepareForUpdate(ctx, validPs, invalidPs)
errs = Strategy.ValidateUpdate(ctx, validPs, ps)
if len(errs) != 0 {
t.Errorf("should ignore updates to PersistentVolumeClaimRetentionPolicyType")
}
})
validPs.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"a": "bar"}} validPs.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"a": "bar"}}
Strategy.PrepareForUpdate(ctx, validPs, ps) Strategy.PrepareForUpdate(ctx, validPs, ps)
errs = Strategy.ValidateUpdate(ctx, validPs, ps) errs = Strategy.ValidateUpdate(ctx, validPs, ps)

View File

@ -339,19 +339,27 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding)
eventsRule(), eventsRule(),
}, },
}) })
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ addControllerRole(&controllerRoles, &controllerRoleBindings, func() rbacv1.ClusterRole {
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "statefulset-controller"}, role := rbacv1.ClusterRole{
Rules: []rbacv1.PolicyRule{ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "statefulset-controller"},
rbacv1helpers.NewRule("list", "watch").Groups(legacyGroup).Resources("pods").RuleOrDie(), Rules: []rbacv1.PolicyRule{
rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), rbacv1helpers.NewRule("list", "watch").Groups(legacyGroup).Resources("pods").RuleOrDie(),
rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/status").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(),
rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/finalizers").RuleOrDie(), rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/status").RuleOrDie(),
rbacv1helpers.NewRule("get", "create", "delete", "update", "patch").Groups(legacyGroup).Resources("pods").RuleOrDie(), rbacv1helpers.NewRule("update").Groups(appsGroup).Resources("statefulsets/finalizers").RuleOrDie(),
rbacv1helpers.NewRule("get", "create", "delete", "update", "patch", "list", "watch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(), rbacv1helpers.NewRule("get", "create", "delete", "update", "patch").Groups(legacyGroup).Resources("pods").RuleOrDie(),
rbacv1helpers.NewRule("get", "create").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(), rbacv1helpers.NewRule("get", "create", "delete", "update", "patch", "list", "watch").Groups(appsGroup).Resources("controllerrevisions").RuleOrDie(),
eventsRule(), rbacv1helpers.NewRule("get", "create").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(),
}, eventsRule(),
}) },
}
if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
role.Rules = append(role.Rules, rbacv1helpers.NewRule("update").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie())
}
return role
}())
addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "ttl-controller"}, ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "ttl-controller"},
Rules: []rbacv1.PolicyRule{ Rules: []rbacv1.PolicyRule{

View File

@ -748,10 +748,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() {
var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() {
*m = StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) {
return fileDescriptor_e1014cab6f31e43b, []int{25}
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int {
return m.Size()
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m)
}
var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo
func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} }
func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) ProtoMessage() {}
func (*StatefulSetSpec) Descriptor() ([]byte, []int) { func (*StatefulSetSpec) Descriptor() ([]byte, []int) {
return fileDescriptor_e1014cab6f31e43b, []int{25} return fileDescriptor_e1014cab6f31e43b, []int{26}
} }
func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -779,7 +809,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo
func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} }
func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) ProtoMessage() {}
func (*StatefulSetStatus) Descriptor() ([]byte, []int) { func (*StatefulSetStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_e1014cab6f31e43b, []int{26} return fileDescriptor_e1014cab6f31e43b, []int{27}
} }
func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -807,7 +837,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo
func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} }
func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) ProtoMessage() {}
func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) {
return fileDescriptor_e1014cab6f31e43b, []int{27} return fileDescriptor_e1014cab6f31e43b, []int{28}
} }
func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -858,6 +888,7 @@ func init() {
proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1.StatefulSet") proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1.StatefulSet")
proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1.StatefulSetCondition") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1.StatefulSetCondition")
proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1.StatefulSetList") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1.StatefulSetList")
proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy")
proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1.StatefulSetSpec") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1.StatefulSetSpec")
proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1.StatefulSetStatus") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1.StatefulSetStatus")
proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1.StatefulSetUpdateStrategy") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1.StatefulSetUpdateStrategy")
@ -868,135 +899,142 @@ func init() {
} }
var fileDescriptor_e1014cab6f31e43b = []byte{ var fileDescriptor_e1014cab6f31e43b = []byte{
// 2047 bytes of a gzipped FileDescriptorProto // 2149 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x24, 0x47, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcb, 0x6f, 0x1b, 0xc7,
0x15, 0x77, 0xcf, 0x87, 0x3d, 0x2e, 0xaf, 0xed, 0xdd, 0xb2, 0xb1, 0x27, 0xbb, 0x64, 0x66, 0x19, 0x19, 0xd7, 0xf2, 0x21, 0x51, 0x23, 0x4b, 0xb2, 0x47, 0xaa, 0xc4, 0xd8, 0x0d, 0xe9, 0xb2, 0xae,
0x60, 0xe3, 0x64, 0xb3, 0x3d, 0xec, 0x66, 0x13, 0xa1, 0x2c, 0x02, 0x79, 0xc6, 0x21, 0x84, 0x78, 0xa3, 0xc4, 0x31, 0x59, 0x3b, 0x4e, 0x50, 0xc4, 0x45, 0x02, 0x91, 0x4a, 0xd3, 0x34, 0x7a, 0x75,
0x6c, 0x53, 0x5e, 0xef, 0x61, 0x09, 0x12, 0xe5, 0xe9, 0xda, 0x71, 0xc7, 0xfd, 0xa5, 0xee, 0xea, 0x64, 0x39, 0x80, 0x9b, 0x16, 0x1d, 0x2d, 0xc7, 0xd4, 0x46, 0xfb, 0xc2, 0xee, 0x2c, 0x63, 0xa2,
0x61, 0x47, 0x5c, 0x10, 0x12, 0x27, 0x38, 0xf0, 0x9f, 0x20, 0x84, 0xc8, 0x0d, 0x45, 0x88, 0xcb, 0x97, 0xa2, 0x40, 0x6f, 0x3d, 0xf4, 0x3f, 0x29, 0x8a, 0xa2, 0xb9, 0x15, 0x41, 0xd0, 0x8b, 0x2f,
0x5e, 0x90, 0x22, 0x2e, 0xe4, 0x64, 0xb1, 0x93, 0x13, 0x42, 0x1c, 0xb9, 0xe4, 0x02, 0xaa, 0xea, 0x45, 0x83, 0x5e, 0x9a, 0x13, 0x51, 0x33, 0xa7, 0xa2, 0xe8, 0xad, 0xbd, 0xf8, 0xd2, 0x62, 0x66,
0xea, 0xef, 0x6a, 0xcf, 0xd8, 0x9b, 0x38, 0x24, 0xca, 0x6d, 0xba, 0xea, 0xf7, 0x7e, 0xf5, 0x5e, 0x67, 0xdf, 0xb3, 0x22, 0x25, 0x27, 0xca, 0x03, 0xbe, 0x89, 0x33, 0xbf, 0xef, 0x37, 0xdf, 0xcc,
0xd5, 0xab, 0x7a, 0xbf, 0xae, 0x1e, 0x70, 0xef, 0xf8, 0xdb, 0x9e, 0xaa, 0xdb, 0xed, 0x63, 0xff, 0x7c, 0xdf, 0x7c, 0xbf, 0x99, 0x15, 0xb8, 0x7d, 0xf4, 0x3d, 0xb7, 0xa9, 0x59, 0xad, 0x23, 0xef,
0x90, 0xb8, 0x16, 0xa1, 0xc4, 0x6b, 0x0f, 0x89, 0xa5, 0xd9, 0x6e, 0x5b, 0x74, 0x60, 0x47, 0x6f, 0x80, 0x38, 0x26, 0xa1, 0xc4, 0x6d, 0xf5, 0x89, 0xd9, 0xb5, 0x9c, 0x96, 0xe8, 0xc0, 0xb6, 0xd6,
0x63, 0xc7, 0xf1, 0xda, 0xc3, 0xdb, 0xed, 0x01, 0xb1, 0x88, 0x8b, 0x29, 0xd1, 0x54, 0xc7, 0xb5, 0xc2, 0xb6, 0xed, 0xb6, 0xfa, 0x37, 0x5a, 0x3d, 0x62, 0x12, 0x07, 0x53, 0xd2, 0x6d, 0xda, 0x8e,
0xa9, 0x0d, 0x61, 0x80, 0x51, 0xb1, 0xa3, 0xab, 0x0c, 0xa3, 0x0e, 0x6f, 0x5f, 0xbd, 0x35, 0xd0, 0x45, 0x2d, 0x08, 0x7d, 0x4c, 0x13, 0xdb, 0x5a, 0x93, 0x61, 0x9a, 0xfd, 0x1b, 0x17, 0xaf, 0xf7,
0xe9, 0x91, 0x7f, 0xa8, 0xf6, 0x6d, 0xb3, 0x3d, 0xb0, 0x07, 0x76, 0x9b, 0x43, 0x0f, 0xfd, 0x47, 0x34, 0x7a, 0xe8, 0x1d, 0x34, 0x55, 0xcb, 0x68, 0xf5, 0xac, 0x9e, 0xd5, 0xe2, 0xd0, 0x03, 0xef,
0xfc, 0x89, 0x3f, 0xf0, 0x5f, 0x01, 0xc5, 0xd5, 0x56, 0x62, 0x98, 0xbe, 0xed, 0x12, 0xc9, 0x30, 0x3e, 0xff, 0xc5, 0x7f, 0xf0, 0xbf, 0x7c, 0x8a, 0x8b, 0x8d, 0xd8, 0x30, 0xaa, 0xe5, 0x10, 0xc9,
0x57, 0xef, 0xc6, 0x18, 0x13, 0xf7, 0x8f, 0x74, 0x8b, 0xb8, 0xa3, 0xb6, 0x73, 0x3c, 0x60, 0x0d, 0x30, 0x17, 0x6f, 0x45, 0x18, 0x03, 0xab, 0x87, 0x9a, 0x49, 0x9c, 0x41, 0xcb, 0x3e, 0xea, 0xb1,
0x5e, 0xdb, 0x24, 0x14, 0xcb, 0xac, 0xda, 0x45, 0x56, 0xae, 0x6f, 0x51, 0xdd, 0x24, 0x39, 0x83, 0x06, 0xb7, 0x65, 0x10, 0x8a, 0x65, 0x56, 0xad, 0x3c, 0x2b, 0xc7, 0x33, 0xa9, 0x66, 0x90, 0x8c,
0xd7, 0x26, 0x19, 0x78, 0xfd, 0x23, 0x62, 0xe2, 0x9c, 0xdd, 0x2b, 0x45, 0x76, 0x3e, 0xd5, 0x8d, 0xc1, 0x2b, 0xe3, 0x0c, 0x5c, 0xf5, 0x90, 0x18, 0x38, 0x63, 0xf7, 0x52, 0x9e, 0x9d, 0x47, 0x35,
0xb6, 0x6e, 0x51, 0x8f, 0xba, 0x59, 0xa3, 0xd6, 0x7f, 0x14, 0x00, 0xbb, 0xb6, 0x45, 0x5d, 0xdb, 0xbd, 0xa5, 0x99, 0xd4, 0xa5, 0x4e, 0xda, 0xa8, 0xf1, 0x5f, 0x05, 0xc0, 0x8e, 0x65, 0x52, 0xc7,
0x30, 0x88, 0x8b, 0xc8, 0x50, 0xf7, 0x74, 0xdb, 0x82, 0x3f, 0x05, 0x35, 0x16, 0x8f, 0x86, 0x29, 0xd2, 0x75, 0xe2, 0x20, 0xd2, 0xd7, 0x5c, 0xcd, 0x32, 0xe1, 0xcf, 0x41, 0x85, 0xcd, 0xa7, 0x8b,
0xae, 0x2b, 0xd7, 0x95, 0x8d, 0x85, 0x3b, 0xdf, 0x52, 0xe3, 0x49, 0x8e, 0xe8, 0x55, 0xe7, 0x78, 0x29, 0xae, 0x2a, 0x97, 0x95, 0xb5, 0xb9, 0x9b, 0xdf, 0x6d, 0x46, 0x8b, 0x1c, 0xd2, 0x37, 0xed,
0xc0, 0x1a, 0x3c, 0x95, 0xa1, 0xd5, 0xe1, 0x6d, 0x75, 0xf7, 0xf0, 0x5d, 0xd2, 0xa7, 0x3d, 0x42, 0xa3, 0x1e, 0x6b, 0x70, 0x9b, 0x0c, 0xdd, 0xec, 0xdf, 0x68, 0xee, 0x1c, 0xbc, 0x47, 0x54, 0xba,
0x71, 0x07, 0x3e, 0x39, 0x69, 0xce, 0x8c, 0x4f, 0x9a, 0x20, 0x6e, 0x43, 0x11, 0x2b, 0xdc, 0x05, 0x45, 0x28, 0x6e, 0xc3, 0x87, 0xc3, 0xfa, 0xd4, 0x68, 0x58, 0x07, 0x51, 0x1b, 0x0a, 0x59, 0xe1,
0x15, 0xce, 0x5e, 0xe2, 0xec, 0xb7, 0x0a, 0xd9, 0x45, 0xd0, 0x2a, 0xc2, 0x3f, 0x7b, 0xe3, 0x31, 0x0e, 0x28, 0x71, 0xf6, 0x02, 0x67, 0xbf, 0x9e, 0xcb, 0x2e, 0x26, 0xdd, 0x44, 0xf8, 0xfd, 0x37,
0x25, 0x16, 0x73, 0xaf, 0x73, 0x49, 0x50, 0x57, 0xb6, 0x30, 0xc5, 0x88, 0x13, 0xc1, 0x97, 0x41, 0x1e, 0x50, 0x62, 0x32, 0xf7, 0xda, 0xe7, 0x04, 0x75, 0x69, 0x03, 0x53, 0x8c, 0x38, 0x11, 0x7c,
0xcd, 0x15, 0xee, 0xd7, 0xcb, 0xd7, 0x95, 0x8d, 0x72, 0xe7, 0xb2, 0x40, 0xd5, 0xc2, 0xb0, 0x50, 0x11, 0x54, 0x1c, 0xe1, 0x7e, 0xb5, 0x78, 0x59, 0x59, 0x2b, 0xb6, 0xcf, 0x0b, 0x54, 0x25, 0x98,
0x84, 0x68, 0xfd, 0x45, 0x01, 0x6b, 0xf9, 0xb8, 0xb7, 0x75, 0x8f, 0xc2, 0x77, 0x72, 0xb1, 0xab, 0x16, 0x0a, 0x11, 0x8d, 0x3f, 0x2b, 0x60, 0x25, 0x3b, 0xef, 0x4d, 0xcd, 0xa5, 0xf0, 0xdd, 0xcc,
0xd3, 0xc5, 0xce, 0xac, 0x79, 0xe4, 0xd1, 0xc0, 0x61, 0x4b, 0x22, 0xee, 0xb7, 0x41, 0x55, 0xa7, 0xdc, 0x9b, 0x93, 0xcd, 0x9d, 0x59, 0xf3, 0x99, 0x87, 0x03, 0x07, 0x2d, 0xb1, 0x79, 0xbf, 0x0d,
0xc4, 0xf4, 0xea, 0xa5, 0xeb, 0xe5, 0x8d, 0x85, 0x3b, 0x37, 0xd4, 0x7c, 0xee, 0xaa, 0x79, 0xc7, 0xca, 0x1a, 0x25, 0x86, 0x5b, 0x2d, 0x5c, 0x2e, 0xae, 0xcd, 0xdd, 0xbc, 0xda, 0xcc, 0xc6, 0x6e,
0x3a, 0x8b, 0x82, 0xb2, 0xfa, 0x16, 0x33, 0x46, 0x01, 0x47, 0xeb, 0xbf, 0x0a, 0x98, 0xdf, 0xc2, 0x33, 0xeb, 0x58, 0x7b, 0x5e, 0x50, 0x96, 0xdf, 0x62, 0xc6, 0xc8, 0xe7, 0x68, 0xfc, 0x4f, 0x01,
0xc4, 0xb4, 0xad, 0x7d, 0x42, 0x2f, 0x60, 0xd1, 0xba, 0xa0, 0xe2, 0x39, 0xa4, 0x2f, 0x16, 0xed, 0xb3, 0x1b, 0x98, 0x18, 0x96, 0xb9, 0x47, 0xe8, 0x19, 0x6c, 0x5a, 0x07, 0x94, 0x5c, 0x9b, 0xa8,
0x6b, 0x32, 0xdf, 0x23, 0x77, 0xf6, 0x1d, 0xd2, 0x8f, 0x17, 0x8a, 0x3d, 0x21, 0x6e, 0x0c, 0xdf, 0x62, 0xd3, 0xbe, 0x25, 0xf3, 0x3d, 0x74, 0x67, 0xcf, 0x26, 0x6a, 0xb4, 0x51, 0xec, 0x17, 0xe2,
0x06, 0xb3, 0x1e, 0xc5, 0xd4, 0xf7, 0xf8, 0x32, 0x2d, 0xdc, 0xf9, 0xfa, 0xe9, 0x34, 0x1c, 0xda, 0xc6, 0xf0, 0x6d, 0x30, 0xed, 0x52, 0x4c, 0x3d, 0x97, 0x6f, 0xd3, 0xdc, 0xcd, 0x6f, 0x1f, 0x4f,
0x59, 0x12, 0x44, 0xb3, 0xc1, 0x33, 0x12, 0x14, 0xad, 0x7f, 0x96, 0x00, 0x8c, 0xb0, 0x5d, 0xdb, 0xc3, 0xa1, 0xed, 0x05, 0x41, 0x34, 0xed, 0xff, 0x46, 0x82, 0xa2, 0xf1, 0xcf, 0x02, 0x80, 0x21,
0xd2, 0x74, 0xca, 0xf2, 0xf7, 0x75, 0x50, 0xa1, 0x23, 0x87, 0xf0, 0x69, 0x98, 0xef, 0xdc, 0x08, 0xb6, 0x63, 0x99, 0x5d, 0x8d, 0xb2, 0xf8, 0x7d, 0x15, 0x94, 0xe8, 0xc0, 0x26, 0x7c, 0x19, 0x66,
0xbd, 0xb8, 0x3f, 0x72, 0xc8, 0xc7, 0x27, 0xcd, 0xb5, 0xbc, 0x05, 0xeb, 0x41, 0xdc, 0x06, 0x6e, 0xdb, 0x57, 0x03, 0x2f, 0xee, 0x0c, 0x6c, 0xf2, 0x78, 0x58, 0x5f, 0xc9, 0x5a, 0xb0, 0x1e, 0xc4,
0x47, 0xfe, 0x95, 0xb8, 0xf5, 0xdd, 0xf4, 0xd0, 0x1f, 0x9f, 0x34, 0x25, 0x87, 0x85, 0x1a, 0x31, 0x6d, 0xe0, 0x66, 0xe8, 0x5f, 0x81, 0x5b, 0xdf, 0x4a, 0x0e, 0xfd, 0x78, 0x58, 0x97, 0x1c, 0x16,
0xa5, 0x1d, 0x84, 0x43, 0x00, 0x0d, 0xec, 0xd1, 0xfb, 0x2e, 0xb6, 0xbc, 0x60, 0x24, 0xdd, 0x24, 0xcd, 0x90, 0x29, 0xe9, 0x20, 0xec, 0x03, 0xa8, 0x63, 0x97, 0xde, 0x71, 0xb0, 0xe9, 0xfa, 0x23,
0x22, 0xf2, 0x97, 0xa6, 0x5b, 0x1e, 0x66, 0xd1, 0xb9, 0x2a, 0xbc, 0x80, 0xdb, 0x39, 0x36, 0x24, 0x69, 0x06, 0x11, 0x33, 0x7f, 0x61, 0xb2, 0xed, 0x61, 0x16, 0xed, 0x8b, 0xc2, 0x0b, 0xb8, 0x99,
0x19, 0x01, 0xde, 0x00, 0xb3, 0x2e, 0xc1, 0x9e, 0x6d, 0xd5, 0x2b, 0x3c, 0x8a, 0x68, 0x02, 0x11, 0x61, 0x43, 0x92, 0x11, 0xe0, 0x55, 0x30, 0xed, 0x10, 0xec, 0x5a, 0x66, 0xb5, 0xc4, 0x67, 0x11,
0x6f, 0x45, 0xa2, 0x17, 0xbe, 0x08, 0xe6, 0x4c, 0xe2, 0x79, 0x78, 0x40, 0xea, 0x55, 0x0e, 0x5c, 0x2e, 0x20, 0xe2, 0xad, 0x48, 0xf4, 0xc2, 0xe7, 0xc1, 0x8c, 0x41, 0x5c, 0x17, 0xf7, 0x48, 0xb5,
0x16, 0xc0, 0xb9, 0x5e, 0xd0, 0x8c, 0xc2, 0xfe, 0xd6, 0xef, 0x15, 0xb0, 0x18, 0xcd, 0xdc, 0x05, 0xcc, 0x81, 0x8b, 0x02, 0x38, 0xb3, 0xe5, 0x37, 0xa3, 0xa0, 0xbf, 0xf1, 0x7b, 0x05, 0xcc, 0x87,
0x6c, 0x95, 0x4e, 0x7a, 0xab, 0x3c, 0x7f, 0x6a, 0x9e, 0x14, 0xec, 0x90, 0xf7, 0xcb, 0x09, 0x9f, 0x2b, 0x77, 0x06, 0xa9, 0xd2, 0x4e, 0xa6, 0xca, 0xb3, 0xc7, 0xc6, 0x49, 0x4e, 0x86, 0x7c, 0x58,
0x59, 0x12, 0xc2, 0x9f, 0x80, 0x9a, 0x47, 0x0c, 0xd2, 0xa7, 0xb6, 0x2b, 0x7c, 0x7e, 0x65, 0x4a, 0x8c, 0xf9, 0xcc, 0x82, 0x10, 0xfe, 0x14, 0x54, 0x5c, 0xa2, 0x13, 0x95, 0x5a, 0x8e, 0xf0, 0xf9,
0x9f, 0xf1, 0x21, 0x31, 0xf6, 0x85, 0x69, 0xe7, 0x12, 0x73, 0x3a, 0x7c, 0x42, 0x11, 0x25, 0xfc, 0xa5, 0x09, 0x7d, 0xc6, 0x07, 0x44, 0xdf, 0x13, 0xa6, 0xed, 0x73, 0xcc, 0xe9, 0xe0, 0x17, 0x0a,
0x11, 0xa8, 0x51, 0x62, 0x3a, 0x06, 0xa6, 0x44, 0x6c, 0x93, 0x54, 0x7e, 0xb3, 0x74, 0x61, 0x64, 0x29, 0xe1, 0x8f, 0x41, 0x85, 0x12, 0xc3, 0xd6, 0x31, 0x25, 0x22, 0x4d, 0x12, 0xf1, 0xcd, 0xc2,
0x7b, 0xb6, 0x76, 0x5f, 0xc0, 0xf8, 0x46, 0x89, 0xe6, 0x21, 0x6c, 0x45, 0x11, 0x0d, 0x3c, 0x06, 0x85, 0x91, 0xed, 0x5a, 0xdd, 0x3b, 0x02, 0xc6, 0x13, 0x25, 0x5c, 0x87, 0xa0, 0x15, 0x85, 0x34,
0x4b, 0xbe, 0xa3, 0x31, 0x24, 0x65, 0x47, 0xf7, 0x60, 0x24, 0xd2, 0xe7, 0xe6, 0xa9, 0x13, 0x72, 0xf0, 0x08, 0x2c, 0x78, 0x76, 0x97, 0x21, 0x29, 0x3b, 0xba, 0x7b, 0x03, 0x11, 0x3e, 0xd7, 0x8e,
0x90, 0x32, 0xe9, 0xac, 0x89, 0x01, 0x96, 0xd2, 0xed, 0x28, 0x43, 0x0d, 0x37, 0xc1, 0xb2, 0xa9, 0x5d, 0x90, 0xfd, 0x84, 0x49, 0x7b, 0x45, 0x0c, 0xb0, 0x90, 0x6c, 0x47, 0x29, 0x6a, 0xb8, 0x0e,
0x5b, 0x88, 0x60, 0x6d, 0xb4, 0x4f, 0xfa, 0xb6, 0xa5, 0x79, 0x3c, 0x81, 0xaa, 0x9d, 0x75, 0x41, 0x16, 0x0d, 0xcd, 0x44, 0x04, 0x77, 0x07, 0x7b, 0x44, 0xb5, 0xcc, 0xae, 0xcb, 0x03, 0xa8, 0xdc,
0xb0, 0xdc, 0x4b, 0x77, 0xa3, 0x2c, 0x1e, 0x6e, 0x83, 0xd5, 0xf0, 0x9c, 0xfd, 0x81, 0xee, 0x51, 0x5e, 0x15, 0x04, 0x8b, 0x5b, 0xc9, 0x6e, 0x94, 0xc6, 0xc3, 0x4d, 0xb0, 0x1c, 0x9c, 0xb3, 0x3f,
0xdb, 0x1d, 0x6d, 0xeb, 0xa6, 0x4e, 0xeb, 0xb3, 0x9c, 0xa7, 0x3e, 0x3e, 0x69, 0xae, 0x22, 0x49, 0xd4, 0x5c, 0x6a, 0x39, 0x83, 0x4d, 0xcd, 0xd0, 0x68, 0x75, 0x9a, 0xf3, 0x54, 0x47, 0xc3, 0xfa,
0x3f, 0x92, 0x5a, 0xb5, 0x7e, 0x33, 0x0b, 0x96, 0x33, 0xa7, 0x01, 0x7c, 0x00, 0xd6, 0xfa, 0xbe, 0x32, 0x92, 0xf4, 0x23, 0xa9, 0x55, 0xe3, 0x37, 0xd3, 0x60, 0x31, 0x75, 0x1a, 0xc0, 0xbb, 0x60,
0xeb, 0x12, 0x8b, 0xee, 0xf8, 0xe6, 0x21, 0x71, 0xf7, 0xfb, 0x47, 0x44, 0xf3, 0x0d, 0xa2, 0xf1, 0x45, 0xf5, 0x1c, 0x87, 0x98, 0x74, 0xdb, 0x33, 0x0e, 0x88, 0xb3, 0xa7, 0x1e, 0x92, 0xae, 0xa7,
0x15, 0xad, 0x76, 0x1a, 0xc2, 0xd7, 0xb5, 0xae, 0x14, 0x85, 0x0a, 0xac, 0xe1, 0x0f, 0x01, 0xb4, 0x93, 0x2e, 0xdf, 0xd1, 0x72, 0xbb, 0x26, 0x7c, 0x5d, 0xe9, 0x48, 0x51, 0x28, 0xc7, 0x1a, 0xfe,
0x78, 0x53, 0x4f, 0xf7, 0xbc, 0x88, 0xb3, 0xc4, 0x39, 0xa3, 0x0d, 0xb8, 0x93, 0x43, 0x20, 0x89, 0x08, 0x40, 0x93, 0x37, 0x6d, 0x69, 0xae, 0x1b, 0x72, 0x16, 0x38, 0x67, 0x98, 0x80, 0xdb, 0x19,
0x15, 0xf3, 0x51, 0x23, 0x9e, 0xee, 0x12, 0x2d, 0xeb, 0x63, 0x39, 0xed, 0xe3, 0x96, 0x14, 0x85, 0x04, 0x92, 0x58, 0x31, 0x1f, 0xbb, 0xc4, 0xd5, 0x1c, 0xd2, 0x4d, 0xfb, 0x58, 0x4c, 0xfa, 0xb8,
0x0a, 0xac, 0xe1, 0xab, 0x60, 0x21, 0x18, 0x8d, 0xcf, 0xb9, 0x58, 0x9c, 0x15, 0x41, 0xb6, 0xb0, 0x21, 0x45, 0xa1, 0x1c, 0x6b, 0xf8, 0x32, 0x98, 0xf3, 0x47, 0xe3, 0x6b, 0x2e, 0x36, 0x67, 0x49,
0x13, 0x77, 0xa1, 0x24, 0x8e, 0x85, 0x66, 0x1f, 0x7a, 0xc4, 0x1d, 0x12, 0xed, 0xcd, 0x40, 0x03, 0x90, 0xcd, 0x6d, 0x47, 0x5d, 0x28, 0x8e, 0x63, 0x53, 0xb3, 0x0e, 0x5c, 0xe2, 0xf4, 0x49, 0xf7,
0xb0, 0x42, 0x59, 0xe5, 0x85, 0x32, 0x0a, 0x6d, 0x37, 0x87, 0x40, 0x12, 0x2b, 0x16, 0x5a, 0x90, 0x4d, 0x5f, 0x03, 0xb0, 0x42, 0x59, 0xe6, 0x85, 0x32, 0x9c, 0xda, 0x4e, 0x06, 0x81, 0x24, 0x56,
0x35, 0xb9, 0xd0, 0x66, 0xd3, 0xa1, 0x1d, 0x48, 0x51, 0xa8, 0xc0, 0x9a, 0xe5, 0x5e, 0xe0, 0xf2, 0x6c, 0x6a, 0x7e, 0xd4, 0x64, 0xa6, 0x36, 0x9d, 0x9c, 0xda, 0xbe, 0x14, 0x85, 0x72, 0xac, 0x59,
0xe6, 0x10, 0xeb, 0x06, 0x3e, 0x34, 0x48, 0x7d, 0x2e, 0x9d, 0x7b, 0x3b, 0xe9, 0x6e, 0x94, 0xc5, 0xec, 0xf9, 0x2e, 0xaf, 0xf7, 0xb1, 0xa6, 0xe3, 0x03, 0x9d, 0x54, 0x67, 0x92, 0xb1, 0xb7, 0x9d,
0xc3, 0x37, 0xc1, 0x95, 0xa0, 0xe9, 0xc0, 0xc2, 0x11, 0x49, 0x8d, 0x93, 0x3c, 0x27, 0x48, 0xae, 0xec, 0x46, 0x69, 0x3c, 0x7c, 0x13, 0x5c, 0xf0, 0x9b, 0xf6, 0x4d, 0x1c, 0x92, 0x54, 0x38, 0xc9,
0xec, 0x64, 0x01, 0x28, 0x6f, 0x03, 0x5f, 0x07, 0x4b, 0x7d, 0xdb, 0x30, 0x78, 0x3e, 0x76, 0x6d, 0x33, 0x82, 0xe4, 0xc2, 0x76, 0x1a, 0x80, 0xb2, 0x36, 0xf0, 0x55, 0xb0, 0xa0, 0x5a, 0xba, 0xce,
0xdf, 0xa2, 0xf5, 0x79, 0xce, 0x02, 0xd9, 0x1e, 0xea, 0xa6, 0x7a, 0x50, 0x06, 0x09, 0x1f, 0x02, 0xe3, 0xb1, 0x63, 0x79, 0x26, 0xad, 0xce, 0x72, 0x16, 0xc8, 0x72, 0xa8, 0x93, 0xe8, 0x41, 0x29,
0xd0, 0x0f, 0xcb, 0x81, 0x57, 0x07, 0xc5, 0x85, 0x3e, 0x5f, 0x87, 0xe2, 0x02, 0x1c, 0x35, 0x79, 0x24, 0xbc, 0x07, 0x80, 0x1a, 0x94, 0x03, 0xb7, 0x0a, 0xf2, 0x0b, 0x7d, 0xb6, 0x0e, 0x45, 0x05,
0x28, 0xc1, 0xd6, 0x7a, 0x5f, 0x01, 0xeb, 0x05, 0x7b, 0x1c, 0x7e, 0x2f, 0x55, 0xf5, 0x6e, 0x66, 0x38, 0x6c, 0x72, 0x51, 0x8c, 0xad, 0xf1, 0xa1, 0x02, 0x56, 0x73, 0x72, 0x1c, 0xbe, 0x9e, 0xa8,
0xaa, 0xde, 0xb5, 0x02, 0xb3, 0x44, 0xe9, 0xeb, 0x83, 0x45, 0xa6, 0x3b, 0x74, 0x6b, 0x10, 0x40, 0x7a, 0xd7, 0x52, 0x55, 0xef, 0x52, 0x8e, 0x59, 0xac, 0xf4, 0xa9, 0x60, 0x9e, 0xe9, 0x0e, 0xcd,
0xc4, 0x09, 0xf6, 0x92, 0xcc, 0x77, 0x94, 0x04, 0xc6, 0xc7, 0xf0, 0x95, 0xf1, 0x49, 0x73, 0x31, 0xec, 0xf9, 0x10, 0x71, 0x82, 0xbd, 0x20, 0xf3, 0x1d, 0xc5, 0x81, 0xd1, 0x31, 0x7c, 0x61, 0x34,
0xd5, 0x87, 0xd2, 0x9c, 0xad, 0x5f, 0x96, 0x00, 0xd8, 0x22, 0x8e, 0x61, 0x8f, 0x4c, 0x62, 0x5d, 0xac, 0xcf, 0x27, 0xfa, 0x50, 0x92, 0xb3, 0xf1, 0xab, 0x02, 0x00, 0x1b, 0xc4, 0xd6, 0xad, 0x81,
0x84, 0x6a, 0xd9, 0x4a, 0xa9, 0x96, 0x96, 0x74, 0x21, 0x22, 0x7f, 0x0a, 0x65, 0xcb, 0x76, 0x46, 0x41, 0xcc, 0xb3, 0x50, 0x2d, 0x1b, 0x09, 0xd5, 0xd2, 0x90, 0x6e, 0x44, 0xe8, 0x4f, 0xae, 0x6c,
0xb6, 0x7c, 0x63, 0x02, 0xcf, 0xe9, 0xba, 0xe5, 0xef, 0x65, 0xb0, 0x12, 0x83, 0x63, 0xe1, 0x72, 0xd9, 0x4c, 0xc9, 0x96, 0x2b, 0x63, 0x78, 0x8e, 0xd7, 0x2d, 0x7f, 0x2f, 0x82, 0xa5, 0x08, 0x1c,
0x2f, 0xb5, 0x84, 0x2f, 0x64, 0x96, 0x70, 0x5d, 0x62, 0xf2, 0xa9, 0x29, 0x97, 0x77, 0xc1, 0x12, 0x09, 0x97, 0xdb, 0x89, 0x2d, 0x7c, 0x2e, 0xb5, 0x85, 0xab, 0x12, 0x93, 0xcf, 0x4d, 0xb9, 0xbc,
0xd3, 0x15, 0xc1, 0xaa, 0x71, 0xd5, 0x32, 0x7b, 0x66, 0xd5, 0x12, 0x55, 0x9d, 0xed, 0x14, 0x13, 0x07, 0x16, 0x98, 0xae, 0xf0, 0x77, 0x8d, 0xab, 0x96, 0xe9, 0x13, 0xab, 0x96, 0xb0, 0xea, 0x6c,
0xca, 0x30, 0x17, 0xa8, 0xa4, 0xb9, 0xcf, 0xa3, 0x4a, 0xfa, 0x83, 0x02, 0x96, 0xe2, 0x65, 0xba, 0x26, 0x98, 0x50, 0x8a, 0x39, 0x47, 0x25, 0xcd, 0x7c, 0x15, 0x55, 0xd2, 0x1f, 0x14, 0xb0, 0x10,
0x00, 0x99, 0xd4, 0x4d, 0xcb, 0xa4, 0xc6, 0xe9, 0x79, 0x59, 0xa0, 0x93, 0xfe, 0x56, 0x49, 0x7a, 0x6d, 0xd3, 0x19, 0xc8, 0xa4, 0x4e, 0x52, 0x26, 0xd5, 0x8e, 0x8f, 0xcb, 0x1c, 0x9d, 0xf4, 0xb7,
0xcd, 0x85, 0xd2, 0x06, 0x7b, 0xa1, 0x72, 0x0c, 0xbd, 0x8f, 0x3d, 0x51, 0x56, 0x2f, 0x05, 0x2f, 0x52, 0xdc, 0x6b, 0x2e, 0x94, 0xd6, 0xd8, 0x85, 0xca, 0xd6, 0x35, 0x15, 0xbb, 0xa2, 0xac, 0x9e,
0x53, 0x41, 0x1b, 0x8a, 0x7a, 0x53, 0x92, 0xaa, 0xf4, 0xe9, 0x4a, 0xaa, 0xf2, 0x27, 0x23, 0xa9, 0xf3, 0x2f, 0x53, 0x7e, 0x1b, 0x0a, 0x7b, 0x13, 0x92, 0xaa, 0xf0, 0xf9, 0x4a, 0xaa, 0xe2, 0x67,
0xee, 0x83, 0x9a, 0x17, 0x8a, 0xa9, 0x0a, 0xa7, 0xbc, 0x31, 0x69, 0x3b, 0x0b, 0x1d, 0x15, 0xb1, 0x23, 0xa9, 0xee, 0x80, 0x8a, 0x1b, 0x88, 0xa9, 0x12, 0xa7, 0xbc, 0x3a, 0x2e, 0x9d, 0x85, 0x8e,
0x46, 0x0a, 0x2a, 0x62, 0x92, 0x69, 0xa7, 0xea, 0x67, 0xa9, 0x9d, 0x58, 0x7a, 0x3b, 0xd8, 0xf7, 0x0a, 0x59, 0x43, 0x05, 0x15, 0x32, 0xc9, 0xb4, 0x53, 0xf9, 0x8b, 0xd4, 0x4e, 0x2c, 0xbc, 0x6d,
0x88, 0xc6, 0xb7, 0x52, 0x2d, 0x4e, 0xef, 0x3d, 0xde, 0x8a, 0x44, 0x2f, 0x3c, 0x00, 0xeb, 0x8e, 0xec, 0xb9, 0xa4, 0xcb, 0x53, 0xa9, 0x12, 0x85, 0xf7, 0x2e, 0x6f, 0x45, 0xa2, 0x17, 0xee, 0x83,
0x6b, 0x0f, 0x5c, 0xe2, 0x79, 0x5b, 0x04, 0x6b, 0x86, 0x6e, 0x91, 0x30, 0x80, 0xa0, 0xea, 0x5d, 0x55, 0xdb, 0xb1, 0x7a, 0x0e, 0x71, 0xdd, 0x0d, 0x82, 0xbb, 0xba, 0x66, 0x92, 0x60, 0x02, 0x7e,
0x1b, 0x9f, 0x34, 0xd7, 0xf7, 0xe4, 0x10, 0x54, 0x64, 0xdb, 0xfa, 0x53, 0x05, 0x5c, 0xce, 0x9e, 0xd5, 0xbb, 0x34, 0x1a, 0xd6, 0x57, 0x77, 0xe5, 0x10, 0x94, 0x67, 0xdb, 0xf8, 0x53, 0x09, 0x9c,
0x88, 0x05, 0x42, 0x44, 0x39, 0x97, 0x10, 0x79, 0x39, 0x91, 0xa2, 0x81, 0x4a, 0x4b, 0xbc, 0xf3, 0x4f, 0x9f, 0x88, 0x39, 0x42, 0x44, 0x39, 0x95, 0x10, 0x79, 0x31, 0x16, 0xa2, 0xbe, 0x4a, 0x8b,
0xe7, 0xd2, 0x74, 0x13, 0x2c, 0x0b, 0xe1, 0x11, 0x76, 0x0a, 0x29, 0x16, 0x2d, 0xcf, 0x41, 0xba, 0xdd, 0xf9, 0x33, 0x61, 0xba, 0x0e, 0x16, 0x85, 0xf0, 0x08, 0x3a, 0x85, 0x14, 0x0b, 0xb7, 0x67,
0x1b, 0x65, 0xf1, 0xf0, 0x1e, 0x58, 0x74, 0xb9, 0xb6, 0x0a, 0x09, 0x02, 0x7d, 0xf2, 0x15, 0x41, 0x3f, 0xd9, 0x8d, 0xd2, 0x78, 0x78, 0x1b, 0xcc, 0x3b, 0x5c, 0x5b, 0x05, 0x04, 0xbe, 0x3e, 0xf9,
0xb0, 0x88, 0x92, 0x9d, 0x28, 0x8d, 0x65, 0xda, 0x24, 0x96, 0x1c, 0x21, 0x41, 0x25, 0xad, 0x4d, 0x86, 0x20, 0x98, 0x47, 0xf1, 0x4e, 0x94, 0xc4, 0x32, 0x6d, 0x12, 0x49, 0x8e, 0x80, 0xa0, 0x94,
0x36, 0xb3, 0x00, 0x94, 0xb7, 0x81, 0x3d, 0xb0, 0xe2, 0x5b, 0x79, 0xaa, 0x20, 0xd7, 0xae, 0x09, 0xd4, 0x26, 0xeb, 0x69, 0x00, 0xca, 0xda, 0xc0, 0x2d, 0xb0, 0xe4, 0x99, 0x59, 0x2a, 0x3f, 0xd6,
0xaa, 0x95, 0x83, 0x3c, 0x04, 0xc9, 0xec, 0xe0, 0x8f, 0x53, 0x72, 0x65, 0x96, 0x9f, 0x22, 0x2f, 0x2e, 0x09, 0xaa, 0xa5, 0xfd, 0x2c, 0x04, 0xc9, 0xec, 0xe0, 0x4f, 0x12, 0x72, 0x65, 0x9a, 0x9f,
0x9c, 0xbe, 0x1d, 0xa6, 0xd6, 0x2b, 0x12, 0x1d, 0x55, 0x9b, 0x56, 0x47, 0xb5, 0xde, 0x53, 0x00, 0x22, 0xcf, 0x1d, 0x9f, 0x0e, 0x13, 0xeb, 0x15, 0x89, 0x8e, 0xaa, 0x4c, 0xaa, 0xa3, 0x1a, 0x1f,
0xcc, 0x6f, 0xc1, 0x89, 0x2f, 0xf7, 0x39, 0x8b, 0x44, 0x89, 0xd4, 0xe4, 0x0a, 0xe7, 0xe6, 0x64, 0x28, 0x00, 0x66, 0x53, 0x70, 0xec, 0xe5, 0x3e, 0x63, 0x11, 0x2b, 0x91, 0x5d, 0xb9, 0xc2, 0xb9,
0x85, 0x13, 0x9f, 0xa0, 0xd3, 0x49, 0x1c, 0x31, 0xbd, 0x17, 0x73, 0x31, 0x33, 0x85, 0xc4, 0x89, 0x36, 0x5e, 0xe1, 0x44, 0x27, 0xe8, 0x64, 0x12, 0x47, 0x2c, 0xef, 0xd9, 0x3c, 0xcc, 0x4c, 0x20,
0xfd, 0x79, 0x36, 0x89, 0x93, 0xe0, 0x39, 0x5d, 0xe2, 0xfc, 0xab, 0x04, 0x56, 0x62, 0xf0, 0xd4, 0x71, 0x22, 0x7f, 0x9e, 0x4c, 0xe2, 0xc4, 0x78, 0x8e, 0x97, 0x38, 0xff, 0x2a, 0x80, 0xa5, 0x08,
0x12, 0x47, 0x62, 0xf2, 0xe5, 0xe5, 0xcc, 0x74, 0xb2, 0x23, 0x9e, 0xba, 0xff, 0x13, 0xd9, 0x11, 0x3c, 0xb1, 0xc4, 0x91, 0x98, 0x3c, 0x7d, 0x9c, 0x99, 0x4c, 0x76, 0x44, 0x4b, 0xf7, 0x25, 0x91,
0x3b, 0x54, 0x20, 0x3b, 0x7e, 0x57, 0x4a, 0x7a, 0x7d, 0x46, 0xd9, 0xf1, 0x09, 0x5c, 0x55, 0x7c, 0x1d, 0x91, 0x43, 0x39, 0xb2, 0xe3, 0x77, 0x85, 0xb8, 0xd7, 0x27, 0x94, 0x1d, 0x9f, 0xc1, 0x53,
0xee, 0x94, 0x4b, 0xeb, 0xcf, 0x65, 0x70, 0x39, 0xbb, 0x05, 0x53, 0x75, 0x50, 0x99, 0x58, 0x07, 0xc5, 0x57, 0x4e, 0xb9, 0x34, 0x3e, 0x2a, 0x82, 0xf3, 0xe9, 0x14, 0x4c, 0xd4, 0x41, 0x65, 0x6c,
0xf7, 0xc0, 0xea, 0x23, 0xdf, 0x30, 0x46, 0x3c, 0x86, 0x44, 0x31, 0x0c, 0x2a, 0xe8, 0x57, 0x85, 0x1d, 0xdc, 0x05, 0xcb, 0xf7, 0x3d, 0x5d, 0x1f, 0xf0, 0x39, 0xc4, 0x8a, 0xa1, 0x5f, 0x41, 0xbf,
0xe5, 0xea, 0xf7, 0x25, 0x18, 0x24, 0xb5, 0xcc, 0x97, 0xc5, 0xca, 0xb3, 0x96, 0xc5, 0xea, 0x39, 0x29, 0x2c, 0x97, 0x7f, 0x20, 0xc1, 0x20, 0xa9, 0x65, 0xb6, 0x2c, 0x96, 0x9e, 0xb4, 0x2c, 0x96,
0xca, 0xa2, 0x5c, 0x59, 0x94, 0xcf, 0xa5, 0x2c, 0xa6, 0xae, 0x89, 0x92, 0xe3, 0x6a, 0xe2, 0x3b, 0x4f, 0x51, 0x16, 0xe5, 0xca, 0xa2, 0x78, 0x2a, 0x65, 0x31, 0x71, 0x4d, 0x94, 0x1c, 0x57, 0x63,
0xfc, 0x58, 0x01, 0x6b, 0xf2, 0xd7, 0x67, 0x68, 0x80, 0x25, 0x13, 0x3f, 0x4e, 0x5e, 0x5e, 0x4c, 0xef, 0xf0, 0x23, 0x05, 0xac, 0xc8, 0xaf, 0xcf, 0x50, 0x07, 0x0b, 0x06, 0x7e, 0x10, 0x7f, 0xbc,
0x2a, 0x18, 0x3e, 0xd5, 0x0d, 0x35, 0xf8, 0xba, 0xa3, 0xbe, 0x65, 0xd1, 0x5d, 0x77, 0x9f, 0xba, 0x18, 0x57, 0x30, 0x3c, 0xaa, 0xe9, 0x4d, 0xff, 0xeb, 0x4e, 0xf3, 0x2d, 0x93, 0xee, 0x38, 0x7b,
0xba, 0x35, 0x08, 0x0a, 0x6c, 0x2f, 0xc5, 0x85, 0x32, 0xdc, 0xf0, 0x21, 0xa8, 0x99, 0xf8, 0xf1, 0xd4, 0xd1, 0xcc, 0x9e, 0x5f, 0x60, 0xb7, 0x12, 0x5c, 0x28, 0xc5, 0x0d, 0xef, 0x81, 0x8a, 0x81,
0xbe, 0xef, 0x0e, 0xc2, 0x42, 0x78, 0xf6, 0x71, 0x78, 0xee, 0xf7, 0x04, 0x0b, 0x8a, 0xf8, 0x5a, 0x1f, 0xec, 0x79, 0x4e, 0x2f, 0x28, 0x84, 0x27, 0x1f, 0x87, 0xc7, 0xfe, 0x96, 0x60, 0x41, 0x21,
0x1f, 0x29, 0x60, 0xbd, 0xa0, 0x82, 0x7e, 0x81, 0xa2, 0xdc, 0x05, 0xd7, 0x53, 0x41, 0xb2, 0x0d, 0x5f, 0xe3, 0x53, 0x05, 0xac, 0xe6, 0x54, 0xd0, 0xaf, 0xd1, 0x2c, 0x77, 0xc0, 0xe5, 0xc4, 0x24,
0x49, 0x1e, 0xf9, 0x06, 0xdf, 0x9b, 0x42, 0xaf, 0xdc, 0x04, 0xf3, 0x0e, 0x76, 0xa9, 0x1e, 0x09, 0x59, 0x42, 0x92, 0xfb, 0x9e, 0xce, 0x73, 0x53, 0xe8, 0x95, 0x6b, 0x60, 0xd6, 0xc6, 0x0e, 0xd5,
0xdd, 0x6a, 0x67, 0x71, 0x7c, 0xd2, 0x9c, 0xdf, 0x0b, 0x1b, 0x51, 0xdc, 0xdf, 0xfa, 0x55, 0x09, 0x42, 0xa1, 0x5b, 0x6e, 0xcf, 0x8f, 0x86, 0xf5, 0xd9, 0xdd, 0xa0, 0x11, 0x45, 0xfd, 0x8d, 0x5f,
0x2c, 0x24, 0x48, 0x2e, 0x40, 0x3b, 0xbc, 0x91, 0xd2, 0x0e, 0xd2, 0xaf, 0x31, 0xc9, 0xa8, 0x8a, 0x17, 0xc0, 0x5c, 0x8c, 0xe4, 0x0c, 0xb4, 0xc3, 0x1b, 0x09, 0xed, 0x20, 0xfd, 0x1a, 0x13, 0x9f,
0xc4, 0x43, 0x2f, 0x23, 0x1e, 0xbe, 0x39, 0x89, 0xe8, 0x74, 0xf5, 0xf0, 0xef, 0x12, 0x58, 0x4d, 0x55, 0x9e, 0x78, 0xd8, 0x4a, 0x89, 0x87, 0xef, 0x8c, 0x23, 0x3a, 0x5e, 0x3d, 0xfc, 0xbb, 0x00,
0xa0, 0x63, 0xf9, 0xf0, 0x9d, 0x94, 0x7c, 0xd8, 0xc8, 0xc8, 0x87, 0xba, 0xcc, 0xe6, 0x4b, 0xfd, 0x96, 0x63, 0xe8, 0x48, 0x3e, 0x7c, 0x3f, 0x21, 0x1f, 0xd6, 0x52, 0xf2, 0xa1, 0x2a, 0xb3, 0x79,
0x30, 0x59, 0x3f, 0xfc, 0x51, 0x01, 0xcb, 0x89, 0xb9, 0xbb, 0x00, 0x01, 0xb1, 0x95, 0x16, 0x10, 0xaa, 0x1f, 0xc6, 0xeb, 0x87, 0x3f, 0x2a, 0x60, 0x31, 0xb6, 0x76, 0x67, 0x20, 0x20, 0x36, 0x92,
0xcd, 0x09, 0xf9, 0x52, 0xa0, 0x20, 0x7e, 0x3d, 0x9b, 0xf2, 0xfb, 0x0b, 0x7f, 0x73, 0xf1, 0x73, 0x02, 0xa2, 0x3e, 0x26, 0x5e, 0x72, 0x14, 0xc4, 0x7f, 0x14, 0xd0, 0x8a, 0xa1, 0x76, 0x89, 0xe3,
0xb0, 0x3a, 0xb4, 0x0d, 0xdf, 0x24, 0x5d, 0x03, 0xeb, 0x66, 0x08, 0x60, 0x15, 0x97, 0x4d, 0xe2, 0x6a, 0x2e, 0x25, 0x26, 0xbd, 0x6b, 0xe9, 0x9e, 0x41, 0x3a, 0x3a, 0xd6, 0x0c, 0x44, 0x58, 0x83,
0x8b, 0x52, 0x7a, 0xe2, 0x7a, 0xba, 0x47, 0x89, 0x45, 0x1f, 0xc4, 0x96, 0x71, 0x9d, 0x7f, 0x20, 0x66, 0x99, 0xbb, 0x96, 0xae, 0xa9, 0x03, 0x88, 0xc1, 0xdc, 0xfb, 0x87, 0xc4, 0xdc, 0x20, 0x3a,
0xa1, 0x43, 0xd2, 0x41, 0xe0, 0xab, 0x60, 0x81, 0x55, 0x4a, 0xbd, 0x4f, 0x76, 0xb0, 0x19, 0xe6, 0xa1, 0xe2, 0x9b, 0xc1, 0x6c, 0xfb, 0xf5, 0xe0, 0x09, 0xfd, 0x9d, 0xa8, 0xeb, 0xf1, 0xb0, 0xbe,
0x54, 0xf4, 0xed, 0x61, 0x3f, 0xee, 0x42, 0x49, 0x1c, 0x3c, 0x02, 0x2b, 0x8e, 0xad, 0xf5, 0xb0, 0x36, 0x09, 0x23, 0x0f, 0xb0, 0x38, 0x27, 0xfc, 0x19, 0x00, 0xec, 0xe7, 0x9e, 0x8a, 0x83, 0x2f,
0x85, 0x07, 0x84, 0x9d, 0xff, 0x7b, 0xb6, 0xa1, 0xf7, 0x47, 0xfc, 0x4e, 0x63, 0xbe, 0xf3, 0x5a, 0x08, 0xb3, 0xed, 0xd7, 0x82, 0x34, 0x7c, 0x27, 0xec, 0x39, 0xd1, 0x00, 0x31, 0xc6, 0xc6, 0x5f,
0xf8, 0xbe, 0xba, 0x97, 0x87, 0xb0, 0xf7, 0x01, 0x49, 0x33, 0xdf, 0xcf, 0x32, 0x4a, 0x68, 0xe6, 0x67, 0x12, 0xdb, 0xf5, 0xb5, 0x7f, 0xb0, 0xf9, 0x05, 0x58, 0xee, 0x47, 0xab, 0x13, 0x00, 0x98,
0x3e, 0x95, 0xcd, 0xe5, 0xfe, 0x5f, 0x20, 0x4b, 0xae, 0x73, 0x7e, 0x2c, 0x2b, 0xba, 0xad, 0xa9, 0xd0, 0x60, 0xb1, 0xf3, 0xbc, 0x94, 0x5e, 0xb6, 0xae, 0x91, 0xbc, 0xb9, 0x2b, 0xa1, 0x43, 0xd2,
0x9d, 0xeb, 0xb6, 0x46, 0xa2, 0x67, 0xe7, 0xcf, 0xa6, 0x67, 0x5b, 0xef, 0x55, 0xc1, 0x95, 0xdc, 0x41, 0xe0, 0xcb, 0x60, 0x8e, 0x09, 0x04, 0x4d, 0x25, 0xdb, 0xd8, 0x08, 0x52, 0x29, 0xfc, 0xe4,
0x19, 0xfb, 0x19, 0x5e, 0xb9, 0xe4, 0x84, 0x61, 0xf9, 0x0c, 0xc2, 0x70, 0x13, 0x2c, 0x8b, 0xef, 0xb2, 0x17, 0x75, 0xa1, 0x38, 0x0e, 0x1e, 0x82, 0x25, 0xdb, 0xea, 0x6e, 0x61, 0x13, 0xf7, 0x08,
0x74, 0x19, 0x5d, 0x19, 0xcd, 0x47, 0x37, 0xdd, 0x8d, 0xb2, 0x78, 0xd9, 0x95, 0x4f, 0xf5, 0x8c, 0x2b, 0x7b, 0xfe, 0x56, 0xf2, 0xa7, 0x9c, 0xd9, 0xf6, 0x2b, 0xc1, 0x35, 0x7d, 0x37, 0x0b, 0x61,
0x57, 0x3e, 0x49, 0x2f, 0xc4, 0xdf, 0x4b, 0x82, 0xc4, 0xcd, 0x7b, 0x21, 0xfe, 0x65, 0x92, 0xc5, 0xd7, 0x20, 0x49, 0x33, 0x0f, 0x02, 0x19, 0x25, 0x34, 0x32, 0x5f, 0x08, 0x67, 0x32, 0xff, 0x56,
0xc3, 0xef, 0x86, 0x59, 0x19, 0x31, 0xcc, 0x71, 0x86, 0x4c, 0x9a, 0x45, 0x04, 0x19, 0xf4, 0x33, 0x21, 0xcb, 0xa9, 0x53, 0x7e, 0x23, 0xcc, 0x7b, 0xa4, 0xaa, 0x9c, 0xea, 0x91, 0x4a, 0x22, 0xe3,
0x7d, 0x8b, 0x7a, 0x47, 0xf2, 0x2d, 0x6a, 0x63, 0xc2, 0x6e, 0x98, 0xfe, 0x76, 0x47, 0xaa, 0xdd, 0x67, 0x4f, 0x28, 0xe3, 0x3f, 0x52, 0xc0, 0x15, 0x7b, 0x82, 0x34, 0xaa, 0x02, 0xbe, 0x2c, 0x9d,
0x17, 0xce, 0xae, 0xdd, 0x5b, 0x7f, 0x55, 0xc0, 0x73, 0x85, 0xfb, 0x11, 0x6e, 0xa6, 0x6a, 0xfe, 0x31, 0xcb, 0x32, 0x49, 0x46, 0xb6, 0xd7, 0x46, 0xc3, 0xfa, 0x95, 0x49, 0x90, 0x68, 0x22, 0xd7,
0xad, 0x4c, 0xcd, 0x7f, 0xbe, 0xd0, 0x30, 0x51, 0xf8, 0x4d, 0xf9, 0xc5, 0xcf, 0xdd, 0x89, 0x17, 0x1a, 0x1f, 0x94, 0xc1, 0x85, 0x4c, 0x79, 0xfc, 0x02, 0x5f, 0xcb, 0x32, 0x9a, 0xbe, 0x78, 0x02,
0x3f, 0x12, 0x45, 0x37, 0xf9, 0x06, 0xa8, 0xb3, 0xf1, 0xe4, 0x69, 0x63, 0xe6, 0x83, 0xa7, 0x8d, 0x4d, 0xbf, 0x0e, 0x16, 0xc5, 0x27, 0xd6, 0xd4, 0x95, 0x20, 0xdc, 0xd3, 0x4e, 0xb2, 0x1b, 0xa5,
0x99, 0x0f, 0x9f, 0x36, 0x66, 0x7e, 0x31, 0x6e, 0x28, 0x4f, 0xc6, 0x0d, 0xe5, 0x83, 0x71, 0x43, 0xf1, 0xb2, 0xd7, 0xba, 0xf2, 0x09, 0x5f, 0xeb, 0xe2, 0x5e, 0x88, 0xff, 0x0c, 0xf2, 0x93, 0x2f,
0xf9, 0x70, 0xdc, 0x50, 0xfe, 0x31, 0x6e, 0x28, 0xbf, 0xfd, 0xa8, 0x31, 0xf3, 0xb0, 0x34, 0xbc, 0xeb, 0x85, 0xf8, 0x07, 0xa1, 0x34, 0x1e, 0xbe, 0x16, 0x64, 0x56, 0xc8, 0x30, 0xc3, 0x19, 0x52,
0xfd, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x69, 0x8a, 0x39, 0xfa, 0x26, 0x00, 0x00, 0xa9, 0x12, 0x12, 0xa4, 0xd0, 0x4f, 0xf4, 0x19, 0xf1, 0x5d, 0xc9, 0x67, 0xc4, 0xb5, 0x31, 0xa1,
0x3b, 0xf9, 0xc3, 0x9c, 0xf4, 0xda, 0x35, 0x77, 0xf2, 0x6b, 0x57, 0xe3, 0x2f, 0x0a, 0x78, 0x26,
0xf7, 0x4c, 0x81, 0xeb, 0x09, 0xb9, 0x76, 0x3d, 0x25, 0xd7, 0x9e, 0xcd, 0x35, 0x8c, 0x69, 0x36,
0x43, 0xfe, 0x66, 0x77, 0x6b, 0xec, 0x9b, 0x9d, 0x44, 0x8c, 0x8f, 0x7f, 0xbc, 0x6b, 0xaf, 0x3d,
0x7c, 0x54, 0x9b, 0xfa, 0xf8, 0x51, 0x6d, 0xea, 0x93, 0x47, 0xb5, 0xa9, 0x5f, 0x8e, 0x6a, 0xca,
0xc3, 0x51, 0x4d, 0xf9, 0x78, 0x54, 0x53, 0x3e, 0x19, 0xd5, 0x94, 0x7f, 0x8c, 0x6a, 0xca, 0x6f,
0x3f, 0xad, 0x4d, 0xdd, 0x2b, 0xf4, 0x6f, 0xfc, 0x3f, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x7f, 0x6b,
0xc0, 0xb5, 0x28, 0x00, 0x00,
} }
func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { func (m *ControllerRevision) Marshal() (dAtA []byte, err error) {
@ -2291,6 +2329,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
i -= len(m.WhenScaled)
copy(dAtA[i:], m.WhenScaled)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled)))
i--
dAtA[i] = 0x12
i -= len(m.WhenDeleted)
copy(dAtA[i:], m.WhenDeleted)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted)))
i--
dAtA[i] = 0xa
return len(dAtA) - i, nil
}
func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) {
size := m.Size() size := m.Size()
dAtA = make([]byte, size) dAtA = make([]byte, size)
@ -2311,6 +2382,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if m.PersistentVolumeClaimRetentionPolicy != nil {
{
size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintGenerated(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x52
}
i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds))
i-- i--
dAtA[i] = 0x48 dAtA[i] = 0x48
@ -2955,6 +3038,19 @@ func (m *StatefulSetList) Size() (n int) {
return n return n
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.WhenDeleted)
n += 1 + l + sovGenerated(uint64(l))
l = len(m.WhenScaled)
n += 1 + l + sovGenerated(uint64(l))
return n
}
func (m *StatefulSetSpec) Size() (n int) { func (m *StatefulSetSpec) Size() (n int) {
if m == nil { if m == nil {
return 0 return 0
@ -2986,6 +3082,10 @@ func (m *StatefulSetSpec) Size() (n int) {
n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit))
} }
n += 1 + sovGenerated(uint64(m.MinReadySeconds)) n += 1 + sovGenerated(uint64(m.MinReadySeconds))
if m.PersistentVolumeClaimRetentionPolicy != nil {
l = m.PersistentVolumeClaimRetentionPolicy.Size()
n += 1 + l + sovGenerated(uint64(l))
}
return n return n
} }
@ -3399,6 +3499,17 @@ func (this *StatefulSetList) String() string {
}, "") }, "")
return s return s
} }
func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`,
`WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`,
`WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`,
`}`,
}, "")
return s
}
func (this *StatefulSetSpec) String() string { func (this *StatefulSetSpec) String() string {
if this == nil { if this == nil {
return "nil" return "nil"
@ -3418,6 +3529,7 @@ func (this *StatefulSetSpec) String() string {
`UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`,
`RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`,
`MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`,
`PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`,
`}`, `}`,
}, "") }, "")
return s return s
@ -7461,6 +7573,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthGenerated
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0
@ -7749,6 +7975,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
break break
} }
} }
case 10:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.PersistentVolumeClaimRetentionPolicy == nil {
m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:]) skippy, err := skipGenerated(dAtA[iNdEx:])

View File

@ -611,6 +611,23 @@ message StatefulSetList {
repeated StatefulSet items = 2; repeated StatefulSet items = 2;
} }
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
message StatefulSetPersistentVolumeClaimRetentionPolicy {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
optional string whenDeleted = 1;
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
optional string whenScaled = 2;
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
message StatefulSetSpec { message StatefulSetSpec {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -677,6 +694,15 @@ message StatefulSetSpec {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
optional int32 minReadySeconds = 9; optional int32 minReadySeconds = 9;
// persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent
// volume claims created from volumeClaimTemplates. By default, all persistent
// volume claims are created as needed and retained until manually deleted. This
// policy allows the lifecycle to be altered, for example by deleting persistent
// volume claims when their stateful set is deleted, or when their pod is scaled
// down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled,
// which is alpha. +optional
optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10;
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -118,6 +118,40 @@ type RollingUpdateStatefulSetStrategy struct {
Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"`
} }
// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
// deleted or scaled down.
type PersistentVolumeClaimRetentionPolicyType string
const (
// RetainPersistentVolumeClaimRetentionPolicyType is the default
// PersistentVolumeClaimRetentionPolicy and specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will not be deleted.
RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
// RetentionPersistentVolumeClaimRetentionPolicyType specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will be deleted in the scenario specified in
// StatefulSetPersistentVolumeClaimRetentionPolicy.
DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
)
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"`
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"`
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
type StatefulSetSpec struct { type StatefulSetSpec struct {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -184,6 +218,15 @@ type StatefulSetSpec struct {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"`
// persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent
// volume claims created from volumeClaimTemplates. By default, all persistent
// volume claims are created as needed and retained until manually deleted. This
// policy allows the lifecycle to be altered, for example by deleting persistent
// volume claims when their stateful set is deleted, or when their pod is scaled
// down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled,
// which is alpha. +optional
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"`
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -323,17 +323,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string {
return map_StatefulSetList return map_StatefulSetList
} }
var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{
"": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
}
func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string {
return map_StatefulSetPersistentVolumeClaimRetentionPolicy
}
var map_StatefulSetSpec = map[string]string{ var map_StatefulSetSpec = map[string]string{
"": "A StatefulSetSpec is the specification of a StatefulSet.", "": "A StatefulSetSpec is the specification of a StatefulSet.",
"replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.",
"selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors",
"template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.",
"volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.",
"serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.",
"podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.",
"updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.",
"revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.",
"minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.",
"persistentVolumeClaimRetentionPolicy": "persistentVolumeClaimRetentionPolicy describes the lifecycle of persistent volume claims created from volumeClaimTemplates. By default, all persistent volume claims are created as needed and retained until manually deleted. This policy allows the lifecycle to be altered, for example by deleting persistent volume claims when their stateful set is deleted, or when their pod is scaled down. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha. +optional",
} }
func (StatefulSetSpec) SwaggerDoc() map[string]string { func (StatefulSetSpec) SwaggerDoc() map[string]string {

View File

@ -688,6 +688,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object {
return nil return nil
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy {
if in == nil {
return nil
}
out := new(StatefulSetPersistentVolumeClaimRetentionPolicy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = *in *out = *in
@ -715,6 +731,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = new(int32) *out = new(int32)
**out = **in **out = **in
} }
if in.PersistentVolumeClaimRetentionPolicy != nil {
in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy
*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
**out = **in
}
return return
} }

View File

@ -553,10 +553,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() {
var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() {
*m = StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) {
return fileDescriptor_2a07313e8f66e805, []int{18}
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int {
return m.Size()
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m)
}
var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo
func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} }
func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) ProtoMessage() {}
func (*StatefulSetSpec) Descriptor() ([]byte, []int) { func (*StatefulSetSpec) Descriptor() ([]byte, []int) {
return fileDescriptor_2a07313e8f66e805, []int{18} return fileDescriptor_2a07313e8f66e805, []int{19}
} }
func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -584,7 +614,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo
func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} }
func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) ProtoMessage() {}
func (*StatefulSetStatus) Descriptor() ([]byte, []int) { func (*StatefulSetStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_2a07313e8f66e805, []int{19} return fileDescriptor_2a07313e8f66e805, []int{20}
} }
func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -612,7 +642,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo
func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} }
func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) ProtoMessage() {}
func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) {
return fileDescriptor_2a07313e8f66e805, []int{20} return fileDescriptor_2a07313e8f66e805, []int{21}
} }
func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -658,6 +688,7 @@ func init() {
proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta1.StatefulSet") proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta1.StatefulSet")
proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta1.StatefulSetCondition") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta1.StatefulSetCondition")
proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta1.StatefulSetList") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta1.StatefulSetList")
proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy")
proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta1.StatefulSetSpec") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta1.StatefulSetSpec")
proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta1.StatefulSetStatus") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta1.StatefulSetStatus")
proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta1.StatefulSetUpdateStrategy") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta1.StatefulSetUpdateStrategy")
@ -668,124 +699,130 @@ func init() {
} }
var fileDescriptor_2a07313e8f66e805 = []byte{ var fileDescriptor_2a07313e8f66e805 = []byte{
// 1869 bytes of a gzipped FileDescriptorProto // 1968 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x24, 0x47, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x1b, 0xc7,
0x15, 0x77, 0x8f, 0x3d, 0xf6, 0xcc, 0x73, 0x3c, 0xde, 0x2d, 0x9b, 0xf5, 0xc4, 0x81, 0xb1, 0x35, 0x15, 0xd7, 0x4a, 0xa2, 0x44, 0x3d, 0x45, 0x94, 0x3d, 0x52, 0x2d, 0x46, 0x69, 0x49, 0x81, 0x35,
0x44, 0x89, 0xf3, 0xe1, 0x9e, 0xac, 0x13, 0xa2, 0x64, 0x17, 0x45, 0x78, 0xbc, 0x4b, 0xb2, 0x91, 0x12, 0xe5, 0x43, 0xcb, 0x58, 0x49, 0x83, 0xc4, 0x2e, 0xdc, 0x8a, 0x92, 0x1b, 0x3b, 0x90, 0x22,
0x8d, 0x9d, 0xb2, 0x1d, 0x44, 0x00, 0x29, 0x35, 0x3d, 0xb5, 0xb3, 0x1d, 0xf7, 0x97, 0xba, 0xab, 0x65, 0x24, 0xc5, 0x68, 0xfa, 0x81, 0x0c, 0xc9, 0x31, 0xb5, 0xd1, 0x7e, 0x61, 0x77, 0xc8, 0x98,
0x87, 0x1d, 0x71, 0xe1, 0x0f, 0x40, 0x4a, 0xce, 0xfc, 0x15, 0xdc, 0x40, 0x70, 0xe3, 0xb4, 0xc7, 0xe8, 0xa5, 0x7f, 0x40, 0x81, 0xf4, 0xdc, 0xbf, 0xa2, 0xb7, 0x16, 0xed, 0xad, 0x87, 0xc2, 0xc7,
0x88, 0x0b, 0x39, 0x59, 0xec, 0xe4, 0x0a, 0x47, 0x2e, 0x2b, 0x21, 0xa1, 0xaa, 0xae, 0xfe, 0xee, 0xa0, 0x97, 0xa6, 0x17, 0xa2, 0x66, 0xae, 0xed, 0xad, 0xbd, 0x18, 0x28, 0x50, 0xcc, 0xec, 0xec,
0xb6, 0xdb, 0x48, 0x6b, 0x09, 0x6e, 0xd3, 0xf5, 0xde, 0xfb, 0xbd, 0x57, 0x55, 0xef, 0xbd, 0x7a, 0xf7, 0xae, 0xb4, 0x2c, 0x60, 0x01, 0xcd, 0x8d, 0x3b, 0xef, 0xbd, 0xdf, 0x9b, 0x99, 0xf7, 0x31,
0xbf, 0x81, 0x1f, 0x9d, 0xbd, 0xe7, 0xa9, 0xba, 0xdd, 0x3b, 0xf3, 0x07, 0xd4, 0xb5, 0x28, 0xa3, 0xef, 0x47, 0xf8, 0xe1, 0xf9, 0xbb, 0xae, 0xaa, 0x59, 0xcd, 0xf3, 0x7e, 0x9b, 0x3a, 0x26, 0x65,
0x5e, 0x6f, 0x4c, 0xad, 0xa1, 0xed, 0xf6, 0xa4, 0x80, 0x38, 0x7a, 0x8f, 0x38, 0x8e, 0xd7, 0x1b, 0xd4, 0x6d, 0x0e, 0xa8, 0xd9, 0xb5, 0x9c, 0xa6, 0x14, 0x10, 0x5b, 0x6b, 0x12, 0xdb, 0x76, 0x9b,
0xdf, 0x1e, 0x50, 0x46, 0x6e, 0xf7, 0x46, 0xd4, 0xa2, 0x2e, 0x61, 0x74, 0xa8, 0x3a, 0xae, 0xcd, 0x83, 0x5b, 0x6d, 0xca, 0xc8, 0xad, 0x66, 0x8f, 0x9a, 0xd4, 0x21, 0x8c, 0x76, 0x55, 0xdb, 0xb1,
0x6c, 0xb4, 0x16, 0x28, 0xaa, 0xc4, 0xd1, 0x55, 0xae, 0xa8, 0x4a, 0xc5, 0xf5, 0xed, 0x91, 0xce, 0x98, 0x85, 0xd6, 0x3c, 0x45, 0x95, 0xd8, 0x9a, 0xca, 0x15, 0x55, 0xa9, 0xb8, 0xbe, 0xd5, 0xd3,
0x1e, 0xf9, 0x03, 0x55, 0xb3, 0xcd, 0xde, 0xc8, 0x1e, 0xd9, 0x3d, 0xa1, 0x3f, 0xf0, 0x1f, 0x8a, 0xd8, 0x59, 0xbf, 0xad, 0x76, 0x2c, 0xa3, 0xd9, 0xb3, 0x7a, 0x56, 0x53, 0xe8, 0xb7, 0xfb, 0x8f,
0x2f, 0xf1, 0x21, 0x7e, 0x05, 0x38, 0xeb, 0xdd, 0x84, 0x43, 0xcd, 0x76, 0x69, 0x6f, 0x9c, 0xf3, 0xc4, 0x97, 0xf8, 0x10, 0xbf, 0x3c, 0x9c, 0xf5, 0x46, 0xc4, 0x61, 0xc7, 0x72, 0x68, 0x73, 0x90,
0xb5, 0xfe, 0x4e, 0xac, 0x63, 0x12, 0xed, 0x91, 0x6e, 0x51, 0x77, 0xd2, 0x73, 0xce, 0x46, 0x7c, 0xf2, 0xb5, 0xfe, 0x76, 0xa8, 0x63, 0x90, 0xce, 0x99, 0x66, 0x52, 0x67, 0xd8, 0xb4, 0xcf, 0x7b,
0xc1, 0xeb, 0x99, 0x94, 0x91, 0x22, 0xab, 0x5e, 0x99, 0x95, 0xeb, 0x5b, 0x4c, 0x37, 0x69, 0xce, 0x7c, 0xc1, 0x6d, 0x1a, 0x94, 0x91, 0x2c, 0xab, 0x66, 0x9e, 0x95, 0xd3, 0x37, 0x99, 0x66, 0xd0,
0xe0, 0xdd, 0xcb, 0x0c, 0x3c, 0xed, 0x11, 0x35, 0x49, 0xce, 0xee, 0xed, 0x32, 0x3b, 0x9f, 0xe9, 0x94, 0xc1, 0x3b, 0x97, 0x19, 0xb8, 0x9d, 0x33, 0x6a, 0x90, 0x94, 0xdd, 0x5b, 0x79, 0x76, 0x7d,
0x46, 0x4f, 0xb7, 0x98, 0xc7, 0xdc, 0xac, 0x51, 0xf7, 0x5f, 0x0a, 0xa0, 0x3d, 0xdb, 0x62, 0xae, 0xa6, 0xe9, 0x4d, 0xcd, 0x64, 0x2e, 0x73, 0x92, 0x46, 0x8d, 0x7f, 0x2b, 0x80, 0x76, 0x2d, 0x93,
0x6d, 0x18, 0xd4, 0xc5, 0x74, 0xac, 0x7b, 0xba, 0x6d, 0xa1, 0xcf, 0xa1, 0xc1, 0xf7, 0x33, 0x24, 0x39, 0x96, 0xae, 0x53, 0x07, 0xd3, 0x81, 0xe6, 0x6a, 0x96, 0x89, 0x3e, 0x85, 0x32, 0x3f, 0x4f,
0x8c, 0xb4, 0x95, 0x4d, 0x65, 0x6b, 0x71, 0xe7, 0x2d, 0x35, 0x3e, 0xe9, 0x08, 0x5e, 0x75, 0xce, 0x97, 0x30, 0x52, 0x55, 0x36, 0x94, 0xcd, 0xc5, 0xed, 0x37, 0xd5, 0xf0, 0xa6, 0x03, 0x78, 0xd5,
0x46, 0x7c, 0xc1, 0x53, 0xb9, 0xb6, 0x3a, 0xbe, 0xad, 0x1e, 0x0e, 0xbe, 0xa0, 0x1a, 0x3b, 0xa0, 0x3e, 0xef, 0xf1, 0x05, 0x57, 0xe5, 0xda, 0xea, 0xe0, 0x96, 0x7a, 0xd8, 0xfe, 0x8c, 0x76, 0xd8,
0x8c, 0xf4, 0xd1, 0x93, 0xf3, 0x8d, 0x99, 0xe9, 0xf9, 0x06, 0xc4, 0x6b, 0x38, 0x42, 0x45, 0x87, 0x01, 0x65, 0xa4, 0x85, 0x9e, 0x8c, 0xea, 0x53, 0xe3, 0x51, 0x1d, 0xc2, 0x35, 0x1c, 0xa0, 0xa2,
0x30, 0x27, 0xd0, 0x6b, 0x02, 0x7d, 0xbb, 0x14, 0x5d, 0x6e, 0x5a, 0xc5, 0xe4, 0x57, 0xf7, 0x1f, 0x43, 0x98, 0x15, 0xe8, 0xd3, 0x02, 0x7d, 0x2b, 0x17, 0x5d, 0x1e, 0x5a, 0xc5, 0xe4, 0xf3, 0x7b,
0x33, 0x6a, 0xf1, 0xf0, 0xfa, 0x2f, 0x48, 0xe8, 0xb9, 0x7b, 0x84, 0x11, 0x2c, 0x80, 0xd0, 0x9b, 0x8f, 0x19, 0x35, 0xf9, 0xf6, 0x5a, 0x2f, 0x48, 0xe8, 0xd9, 0x3d, 0xc2, 0x08, 0x16, 0x40, 0xe8,
0xd0, 0x70, 0x65, 0xf8, 0xed, 0xd9, 0x4d, 0x65, 0x6b, 0xb6, 0x7f, 0x43, 0x6a, 0x35, 0xc2, 0x6d, 0x0d, 0x28, 0x3b, 0x72, 0xfb, 0xd5, 0x99, 0x0d, 0x65, 0x73, 0xa6, 0x75, 0x4d, 0x6a, 0x95, 0xfd,
0xe1, 0x48, 0xa3, 0xfb, 0x44, 0x81, 0x5b, 0xf9, 0x7d, 0xef, 0xeb, 0x1e, 0x43, 0xbf, 0xc8, 0xed, 0x63, 0xe1, 0x40, 0xa3, 0xf1, 0x44, 0x81, 0x1b, 0xe9, 0x73, 0xef, 0x6b, 0x2e, 0x43, 0x3f, 0x4d,
0x5d, 0xad, 0xb6, 0x77, 0x6e, 0x2d, 0x76, 0x1e, 0x39, 0x0e, 0x57, 0x12, 0xfb, 0x3e, 0x82, 0xba, 0x9d, 0x5d, 0x2d, 0x76, 0x76, 0x6e, 0x2d, 0x4e, 0x1e, 0x38, 0xf6, 0x57, 0x22, 0xe7, 0x3e, 0x82,
0xce, 0xa8, 0xe9, 0xb5, 0x6b, 0x9b, 0xb3, 0x5b, 0x8b, 0x3b, 0x6f, 0xa8, 0x25, 0x09, 0xac, 0xe6, 0x92, 0xc6, 0xa8, 0xe1, 0x56, 0xa7, 0x37, 0x66, 0x36, 0x17, 0xb7, 0x5f, 0x57, 0x73, 0x12, 0x58,
0xa3, 0xeb, 0x2f, 0x49, 0xdc, 0xfa, 0x03, 0x8e, 0x80, 0x03, 0xa0, 0xee, 0x6f, 0x6b, 0x00, 0xf7, 0x4d, 0xef, 0xae, 0xb5, 0x24, 0x71, 0x4b, 0x0f, 0x38, 0x02, 0xf6, 0x80, 0x1a, 0xbf, 0x9a, 0x06,
0xa8, 0x63, 0xd8, 0x13, 0x93, 0x5a, 0xec, 0x1a, 0xae, 0xee, 0x01, 0xcc, 0x79, 0x0e, 0xd5, 0xe4, 0xd8, 0xa3, 0xb6, 0x6e, 0x0d, 0x0d, 0x6a, 0xb2, 0x2b, 0x08, 0xdd, 0x03, 0x98, 0x75, 0x6d, 0xda,
0xd5, 0xbd, 0x5a, 0xba, 0x83, 0x38, 0xa8, 0x63, 0x87, 0x6a, 0xf1, 0xa5, 0xf1, 0x2f, 0x2c, 0x20, 0x91, 0xa1, 0x7b, 0x25, 0xf7, 0x04, 0xe1, 0xa6, 0x8e, 0x6d, 0xda, 0x09, 0x83, 0xc6, 0xbf, 0xb0,
0xd0, 0x27, 0x30, 0xef, 0x31, 0xc2, 0x7c, 0x4f, 0x5c, 0xd9, 0xe2, 0xce, 0x6b, 0x55, 0xc0, 0x84, 0x80, 0x40, 0x1f, 0xc1, 0x9c, 0xcb, 0x08, 0xeb, 0xbb, 0x22, 0x64, 0x8b, 0xdb, 0xaf, 0x16, 0x01,
0x41, 0xbf, 0x25, 0xe1, 0xe6, 0x83, 0x6f, 0x2c, 0x81, 0xba, 0x7f, 0x9b, 0x85, 0x95, 0x58, 0x79, 0x13, 0x06, 0xad, 0x8a, 0x84, 0x9b, 0xf3, 0xbe, 0xb1, 0x04, 0x6a, 0xfc, 0x75, 0x06, 0x56, 0x42,
0xcf, 0xb6, 0x86, 0x3a, 0xe3, 0x29, 0x7d, 0x17, 0xe6, 0xd8, 0xc4, 0xa1, 0xe2, 0x4c, 0x9a, 0xfd, 0xe5, 0x5d, 0xcb, 0xec, 0x6a, 0x8c, 0xa7, 0xf4, 0x1d, 0x98, 0x65, 0x43, 0x9b, 0x8a, 0x3b, 0x59,
0x57, 0xc3, 0x60, 0x4e, 0x26, 0x0e, 0x7d, 0x76, 0xbe, 0xb1, 0x56, 0x60, 0xc2, 0x45, 0x58, 0x18, 0x68, 0xbd, 0xe2, 0x6f, 0xe6, 0x64, 0x68, 0xd3, 0x67, 0xa3, 0xfa, 0x5a, 0x86, 0x09, 0x17, 0x61,
0xa1, 0xfd, 0x28, 0xce, 0x9a, 0x30, 0x7f, 0x27, 0xed, 0xfc, 0xd9, 0xf9, 0x46, 0x41, 0x03, 0x51, 0x61, 0x84, 0xf6, 0x83, 0x7d, 0x4e, 0x0b, 0xf3, 0xb7, 0xe3, 0xce, 0x9f, 0x8d, 0xea, 0x19, 0x0d,
0x23, 0xa4, 0x74, 0x88, 0xe8, 0x0b, 0x68, 0x19, 0xc4, 0x63, 0xa7, 0xce, 0x90, 0x30, 0x7a, 0xa2, 0x44, 0x0d, 0x90, 0xe2, 0x5b, 0x44, 0x9f, 0x41, 0x45, 0x27, 0x2e, 0x3b, 0xb5, 0xbb, 0x84, 0xd1,
0x9b, 0xb4, 0x3d, 0x2f, 0x76, 0xff, 0x7a, 0xb5, 0x8b, 0xe2, 0x16, 0xfd, 0x5b, 0x32, 0x82, 0xd6, 0x13, 0xcd, 0xa0, 0xd5, 0x39, 0x71, 0xfa, 0xd7, 0x8a, 0x05, 0x8a, 0x5b, 0xb4, 0x6e, 0xc8, 0x1d,
0x7e, 0x0a, 0x09, 0x67, 0x90, 0xd1, 0x18, 0x10, 0x5f, 0x39, 0x71, 0x89, 0xe5, 0x05, 0xbb, 0xe2, 0x54, 0xf6, 0x63, 0x48, 0x38, 0x81, 0x8c, 0x06, 0x80, 0xf8, 0xca, 0x89, 0x43, 0x4c, 0xd7, 0x3b,
0xfe, 0x16, 0xae, 0xec, 0x6f, 0x5d, 0xfa, 0x43, 0xfb, 0x39, 0x34, 0x5c, 0xe0, 0x01, 0xbd, 0x02, 0x15, 0xf7, 0x37, 0x3f, 0xb1, 0xbf, 0x75, 0xe9, 0x0f, 0xed, 0xa7, 0xd0, 0x70, 0x86, 0x07, 0xf4,
0xf3, 0x2e, 0x25, 0x9e, 0x6d, 0xb5, 0xe7, 0xc4, 0x89, 0x45, 0xd7, 0x85, 0xc5, 0x2a, 0x96, 0x52, 0x32, 0xcc, 0x39, 0x94, 0xb8, 0x96, 0x59, 0x9d, 0x15, 0x37, 0x16, 0x84, 0x0b, 0x8b, 0x55, 0x2c,
0xf4, 0x1a, 0x2c, 0x98, 0xd4, 0xf3, 0xc8, 0x88, 0xb6, 0xeb, 0x42, 0x71, 0x59, 0x2a, 0x2e, 0x1c, 0xa5, 0xe8, 0x55, 0x98, 0x37, 0xa8, 0xeb, 0x92, 0x1e, 0xad, 0x96, 0x84, 0xe2, 0xb2, 0x54, 0x9c,
0x04, 0xcb, 0x38, 0x94, 0x77, 0xff, 0xa8, 0x40, 0x2b, 0xbe, 0xa6, 0x6b, 0xa8, 0xd5, 0x8f, 0xd2, 0x3f, 0xf0, 0x96, 0xb1, 0x2f, 0x6f, 0xfc, 0x5e, 0x81, 0x4a, 0x18, 0xa6, 0x2b, 0xa8, 0xd5, 0xfb,
0xb5, 0xfa, 0xfd, 0x0a, 0xc9, 0x59, 0x52, 0xa3, 0xff, 0xa8, 0x01, 0x8a, 0x95, 0xb0, 0x6d, 0x18, 0xf1, 0x5a, 0xfd, 0x6e, 0x81, 0xe4, 0xcc, 0xa9, 0xd1, 0x7f, 0x4c, 0x03, 0x0a, 0x95, 0xb0, 0xa5,
0x03, 0xa2, 0x9d, 0xa1, 0x4d, 0x98, 0xb3, 0x88, 0x19, 0xe6, 0x64, 0x54, 0x20, 0x3f, 0x21, 0x26, 0xeb, 0x6d, 0xd2, 0x39, 0x47, 0x1b, 0x30, 0x6b, 0x12, 0xc3, 0xcf, 0xc9, 0xa0, 0x40, 0x3e, 0x24,
0xc5, 0x42, 0x82, 0xbe, 0x54, 0x00, 0xf9, 0xe2, 0x36, 0x87, 0xbb, 0x96, 0x65, 0x33, 0xc2, 0x0f, 0x06, 0xc5, 0x42, 0x82, 0xbe, 0x50, 0x00, 0xf5, 0x45, 0x34, 0xbb, 0x3b, 0xa6, 0x69, 0x31, 0xc2,
0x38, 0x0c, 0x68, 0xaf, 0x42, 0x40, 0xa1, 0x2f, 0xf5, 0x34, 0x87, 0x72, 0xdf, 0x62, 0xee, 0x24, 0x2f, 0xd8, 0xdf, 0xd0, 0x6e, 0x81, 0x0d, 0xf9, 0xbe, 0xd4, 0xd3, 0x14, 0xca, 0x3d, 0x93, 0x39,
0xbe, 0xd8, 0xbc, 0x02, 0x2e, 0x70, 0x8d, 0x7e, 0x0e, 0xe0, 0x4a, 0xcc, 0x13, 0x5b, 0x96, 0x6d, 0xc3, 0x30, 0xb0, 0x69, 0x05, 0x9c, 0xe1, 0x1a, 0xfd, 0x04, 0xc0, 0x91, 0x98, 0x27, 0x96, 0x2c,
0x79, 0x0f, 0x08, 0xdd, 0xef, 0xd9, 0xd6, 0x43, 0x7d, 0x14, 0x37, 0x16, 0x1c, 0x41, 0xe0, 0x04, 0xdb, 0xfc, 0x1e, 0xe0, 0xbb, 0xdf, 0xb5, 0xcc, 0x47, 0x5a, 0x2f, 0x6c, 0x2c, 0x38, 0x80, 0xc0,
0xdc, 0xfa, 0x7d, 0x58, 0x2b, 0x89, 0x13, 0xdd, 0x80, 0xd9, 0x33, 0x3a, 0x09, 0x8e, 0x0a, 0xf3, 0x11, 0xb8, 0xf5, 0x7b, 0xb0, 0x96, 0xb3, 0x4f, 0x74, 0x0d, 0x66, 0xce, 0xe9, 0xd0, 0xbb, 0x2a,
0x9f, 0x68, 0x15, 0xea, 0x63, 0x62, 0xf8, 0x34, 0xa8, 0x49, 0x1c, 0x7c, 0xdc, 0xa9, 0xbd, 0xa7, 0xcc, 0x7f, 0xa2, 0x55, 0x28, 0x0d, 0x88, 0xde, 0xa7, 0x5e, 0x4d, 0x62, 0xef, 0xe3, 0xf6, 0xf4,
0x74, 0x7f, 0x5f, 0x4f, 0x66, 0x0a, 0xef, 0x37, 0x68, 0x8b, 0x3f, 0x0f, 0x8e, 0xa1, 0x6b, 0xc4, 0xbb, 0x4a, 0xe3, 0xb7, 0xa5, 0x68, 0xa6, 0xf0, 0x7e, 0x83, 0x36, 0xf9, 0xf3, 0x60, 0xeb, 0x5a,
0x13, 0x18, 0xf5, 0xfe, 0x0b, 0xc1, 0xd3, 0x10, 0xac, 0xe1, 0x48, 0x8a, 0x7e, 0x09, 0x0d, 0x8f, 0x87, 0xb8, 0x02, 0xa3, 0xd4, 0x7a, 0xc1, 0x7b, 0x1a, 0xbc, 0x35, 0x1c, 0x48, 0xd1, 0xcf, 0xa0,
0x1a, 0x54, 0x63, 0xb6, 0x2b, 0x5b, 0xdc, 0xdb, 0x15, 0x73, 0x8a, 0x0c, 0xa8, 0x71, 0x2c, 0x4d, 0xec, 0x52, 0x9d, 0x76, 0x98, 0xe5, 0xc8, 0x16, 0xf7, 0x56, 0xc1, 0x9c, 0x22, 0x6d, 0xaa, 0x1f,
0x03, 0xf8, 0xf0, 0x0b, 0x47, 0x90, 0xe8, 0x13, 0x68, 0x30, 0x6a, 0x3a, 0x06, 0x61, 0x54, 0x9e, 0x4b, 0x53, 0x0f, 0xde, 0xff, 0xc2, 0x01, 0x24, 0xfa, 0x08, 0xca, 0x8c, 0x1a, 0xb6, 0x4e, 0x18,
0x5e, 0x2a, 0xaf, 0x78, 0xef, 0xe0, 0x60, 0x47, 0xf6, 0xf0, 0x44, 0xaa, 0x89, 0xee, 0x19, 0xe5, 0x95, 0xb7, 0x17, 0xcb, 0x2b, 0xde, 0x3b, 0x38, 0xd8, 0x91, 0xd5, 0x3d, 0x91, 0x6a, 0xa2, 0x7b,
0x69, 0xb8, 0x8a, 0x23, 0x18, 0xf4, 0x33, 0x68, 0x78, 0x8c, 0xbf, 0xea, 0xa3, 0x89, 0xa8, 0xb6, 0x06, 0x79, 0xea, 0xaf, 0xe2, 0x00, 0x06, 0xfd, 0x18, 0xca, 0x2e, 0xe3, 0xaf, 0x7a, 0x6f, 0x28,
0x8b, 0x9e, 0x95, 0x64, 0x1f, 0x0d, 0x4c, 0x62, 0xe8, 0x70, 0x05, 0x47, 0x70, 0x68, 0x17, 0x96, 0xaa, 0xed, 0xa2, 0x67, 0x25, 0xda, 0x47, 0x3d, 0x93, 0x10, 0xda, 0x5f, 0xc1, 0x01, 0x1c, 0xda,
0x4d, 0xdd, 0xc2, 0x94, 0x0c, 0x27, 0xc7, 0x54, 0xb3, 0xad, 0xa1, 0x27, 0xca, 0xb4, 0xde, 0x5f, 0x81, 0x65, 0x43, 0x33, 0x31, 0x25, 0xdd, 0xe1, 0x31, 0xed, 0x58, 0x66, 0xd7, 0x15, 0x65, 0x5a,
0x93, 0x46, 0xcb, 0x07, 0x69, 0x31, 0xce, 0xea, 0xa3, 0x7d, 0x58, 0x0d, 0x9f, 0xdd, 0x8f, 0x74, 0x6a, 0xad, 0x49, 0xa3, 0xe5, 0x83, 0xb8, 0x18, 0x27, 0xf5, 0xd1, 0x3e, 0xac, 0xfa, 0xcf, 0xee,
0x8f, 0xd9, 0xee, 0x64, 0x5f, 0x37, 0x75, 0x26, 0x7a, 0x5e, 0xbd, 0xdf, 0x9e, 0x9e, 0x6f, 0xac, 0x7d, 0xcd, 0x65, 0x96, 0x33, 0xdc, 0xd7, 0x0c, 0x8d, 0x89, 0x9e, 0x57, 0x6a, 0x55, 0xc7, 0xa3,
0xe2, 0x02, 0x39, 0x2e, 0xb4, 0xe2, 0x7d, 0xc5, 0x21, 0xbe, 0x47, 0x87, 0xa2, 0x87, 0x35, 0xe2, 0xfa, 0x2a, 0xce, 0x90, 0xe3, 0x4c, 0x2b, 0xde, 0x57, 0x6c, 0xd2, 0x77, 0x69, 0x57, 0xf4, 0xb0,
0xbe, 0x72, 0x24, 0x56, 0xb1, 0x94, 0xa2, 0x9f, 0xa6, 0xd2, 0xb4, 0x71, 0xb5, 0x34, 0x6d, 0x95, 0x72, 0xd8, 0x57, 0x8e, 0xc4, 0x2a, 0x96, 0x52, 0xf4, 0x30, 0x96, 0xa6, 0xe5, 0xc9, 0xd2, 0xb4,
0xa7, 0x28, 0x3a, 0x85, 0x35, 0xc7, 0xb5, 0x47, 0x2e, 0xf5, 0xbc, 0x7b, 0x94, 0x0c, 0x0d, 0xdd, 0x92, 0x9f, 0xa2, 0xe8, 0x14, 0xd6, 0x6c, 0xc7, 0xea, 0x39, 0xd4, 0x75, 0xf7, 0x28, 0xe9, 0xea,
0xa2, 0xe1, 0xc9, 0x34, 0xc5, 0x8e, 0x5e, 0x9a, 0x9e, 0x6f, 0xac, 0x1d, 0x15, 0xab, 0xe0, 0x32, 0x9a, 0x49, 0xfd, 0x9b, 0x59, 0x10, 0x27, 0x7a, 0x69, 0x3c, 0xaa, 0xaf, 0x1d, 0x65, 0xab, 0xe0,
0xdb, 0xee, 0x5f, 0xe6, 0xe0, 0x46, 0xf6, 0x8d, 0x43, 0x1f, 0x03, 0xb2, 0x07, 0x1e, 0x75, 0xc7, 0x3c, 0xdb, 0xc6, 0x9f, 0x66, 0xe1, 0x5a, 0xf2, 0x8d, 0x43, 0x1f, 0x00, 0xb2, 0xda, 0x2e, 0x75,
0x74, 0xf8, 0x61, 0x30, 0xb8, 0xf1, 0xe9, 0x46, 0x11, 0xd3, 0x4d, 0x54, 0xb7, 0x87, 0x39, 0x0d, 0x06, 0xb4, 0xfb, 0xbe, 0x37, 0xb8, 0xf1, 0xe9, 0x46, 0x11, 0xd3, 0x4d, 0x50, 0xb7, 0x87, 0x29,
0x5c, 0x60, 0x15, 0xcc, 0x47, 0xb2, 0x00, 0x6a, 0x22, 0xd0, 0xc4, 0x7c, 0x94, 0x2b, 0x82, 0x5d, 0x0d, 0x9c, 0x61, 0xe5, 0xcd, 0x47, 0xb2, 0x00, 0xa6, 0xc5, 0x46, 0x23, 0xf3, 0x51, 0xaa, 0x08,
0x58, 0x96, 0xb5, 0x1f, 0x0a, 0x45, 0xb2, 0x26, 0xee, 0xfd, 0x34, 0x2d, 0xc6, 0x59, 0x7d, 0x74, 0x76, 0x60, 0x59, 0xd6, 0xbe, 0x2f, 0x14, 0xc9, 0x1a, 0x89, 0xfb, 0x69, 0x5c, 0x8c, 0x93, 0xfa,
0x17, 0x96, 0x5c, 0x9e, 0x07, 0x11, 0xc0, 0x82, 0x00, 0xf8, 0x8e, 0x04, 0x58, 0xc2, 0x49, 0x21, 0xe8, 0x0e, 0x2c, 0x39, 0x3c, 0x0f, 0x02, 0x80, 0x79, 0x01, 0xf0, 0x2d, 0x09, 0xb0, 0x84, 0xa3,
0x4e, 0xeb, 0xa2, 0x0f, 0xe1, 0x26, 0x19, 0x13, 0xdd, 0x20, 0x03, 0x83, 0x46, 0x00, 0x73, 0x02, 0x42, 0x1c, 0xd7, 0x45, 0xef, 0xc3, 0x75, 0x32, 0x20, 0x9a, 0x4e, 0xda, 0x3a, 0x0d, 0x00, 0x66,
0xe0, 0x45, 0x09, 0x70, 0x73, 0x37, 0xab, 0x80, 0xf3, 0x36, 0xe8, 0x00, 0x56, 0x7c, 0x2b, 0x0f, 0x05, 0xc0, 0x8b, 0x12, 0xe0, 0xfa, 0x4e, 0x52, 0x01, 0xa7, 0x6d, 0xd0, 0x01, 0xac, 0xf4, 0xcd,
0x15, 0x24, 0xf1, 0x4b, 0x12, 0x6a, 0xe5, 0x34, 0xaf, 0x82, 0x8b, 0xec, 0xd0, 0xe7, 0x00, 0x5a, 0x34, 0x94, 0x97, 0xc4, 0x2f, 0x49, 0xa8, 0x95, 0xd3, 0xb4, 0x0a, 0xce, 0xb2, 0x43, 0x9f, 0x02,
0xf8, 0xaa, 0x7b, 0xed, 0x79, 0xd1, 0x86, 0xdf, 0xac, 0x50, 0x6c, 0xd1, 0x28, 0x10, 0xb7, 0xc0, 0x74, 0xfc, 0x57, 0xdd, 0xad, 0xce, 0x89, 0x36, 0xfc, 0x46, 0x81, 0x62, 0x0b, 0x46, 0x81, 0xb0,
0x68, 0xc9, 0xc3, 0x09, 0x4c, 0x74, 0x07, 0x5a, 0x9a, 0x6d, 0x18, 0x22, 0xf3, 0xf7, 0x6c, 0xdf, 0x05, 0x06, 0x4b, 0x2e, 0x8e, 0x60, 0xa2, 0xdb, 0x50, 0xe9, 0x58, 0xba, 0x2e, 0x32, 0x7f, 0xd7,
0x62, 0x22, 0x79, 0xeb, 0x7d, 0xc4, 0x1f, 0xfb, 0xbd, 0x94, 0x04, 0x67, 0x34, 0xbb, 0x7f, 0x56, 0xea, 0x9b, 0x4c, 0x24, 0x6f, 0xa9, 0x85, 0xf8, 0x63, 0xbf, 0x1b, 0x93, 0xe0, 0x84, 0x66, 0xe3,
0x92, 0xcf, 0x4c, 0x58, 0xce, 0xe8, 0x4e, 0x6a, 0xf4, 0x79, 0x25, 0x33, 0xfa, 0xdc, 0xca, 0x5b, 0x8f, 0x4a, 0xf4, 0x99, 0xf1, 0xcb, 0x19, 0xdd, 0x8e, 0x8d, 0x3e, 0x2f, 0x27, 0x46, 0x9f, 0x1b,
0x24, 0x26, 0x1f, 0x1d, 0x96, 0x78, 0xf2, 0xeb, 0xd6, 0x28, 0xb8, 0x70, 0xd9, 0x12, 0xdf, 0xba, 0x69, 0x8b, 0xc8, 0xe4, 0xa3, 0xc1, 0x12, 0x4f, 0x7e, 0xcd, 0xec, 0x79, 0x01, 0x97, 0x2d, 0xf1,
0xb0, 0x94, 0x22, 0xed, 0xc4, 0xc3, 0x78, 0x53, 0xdc, 0x79, 0x52, 0x88, 0xd3, 0xc8, 0xdd, 0x0f, 0xcd, 0x0b, 0x4b, 0x29, 0xd0, 0x8e, 0x3c, 0x8c, 0xd7, 0x45, 0xcc, 0xa3, 0x42, 0x1c, 0x47, 0x6e,
0xa0, 0x95, 0xae, 0xc3, 0xd4, 0x4c, 0xaf, 0x5c, 0x3a, 0xd3, 0x7f, 0xab, 0xc0, 0x5a, 0x89, 0x77, 0xdc, 0x85, 0x4a, 0xbc, 0x0e, 0x63, 0x33, 0xbd, 0x72, 0xe9, 0x4c, 0xff, 0xb5, 0x02, 0x6b, 0x39,
0x64, 0x40, 0xcb, 0x24, 0x8f, 0x13, 0xd7, 0x7c, 0xe9, 0x6c, 0xcc, 0x59, 0x93, 0x1a, 0xb0, 0x26, 0xde, 0x91, 0x0e, 0x15, 0x83, 0x3c, 0x8e, 0x84, 0xf9, 0xd2, 0xd9, 0x98, 0xb3, 0x26, 0xd5, 0x63,
0xf5, 0x81, 0xc5, 0x0e, 0xdd, 0x63, 0xe6, 0xea, 0xd6, 0x28, 0xb8, 0x87, 0x83, 0x14, 0x16, 0xce, 0x4d, 0xea, 0x03, 0x93, 0x1d, 0x3a, 0xc7, 0xcc, 0xd1, 0xcc, 0x9e, 0x17, 0x87, 0x83, 0x18, 0x16,
0x60, 0xa3, 0xcf, 0xa0, 0x61, 0x92, 0xc7, 0xc7, 0xbe, 0x3b, 0x2a, 0x3a, 0xaf, 0x6a, 0x7e, 0xc4, 0x4e, 0x60, 0xa3, 0x4f, 0xa0, 0x6c, 0x90, 0xc7, 0xc7, 0x7d, 0xa7, 0x97, 0x75, 0x5f, 0xc5, 0xfc,
0xfb, 0x71, 0x20, 0x51, 0x70, 0x84, 0xd7, 0x3d, 0x84, 0xcd, 0xd4, 0x26, 0x79, 0xab, 0xa0, 0x0f, 0x88, 0xf7, 0xe3, 0x40, 0xa2, 0xe0, 0x00, 0xaf, 0x71, 0x08, 0x1b, 0xb1, 0x43, 0xf2, 0x56, 0x41,
0x7d, 0xe3, 0x98, 0xc6, 0x17, 0xfe, 0x06, 0x34, 0x1d, 0xe2, 0x32, 0x3d, 0x6a, 0x17, 0xf5, 0xfe, 0x1f, 0xf5, 0xf5, 0x63, 0x1a, 0x06, 0xfc, 0x75, 0x58, 0xb0, 0x89, 0xc3, 0xb4, 0xa0, 0x5d, 0x94,
0xd2, 0xf4, 0x7c, 0xa3, 0x79, 0x14, 0x2e, 0xe2, 0x58, 0xde, 0xfd, 0xb7, 0x02, 0xf5, 0x63, 0x8d, 0x5a, 0x4b, 0xe3, 0x51, 0x7d, 0xe1, 0xc8, 0x5f, 0xc4, 0xa1, 0xbc, 0xf1, 0x1f, 0x05, 0x4a, 0xc7,
0x18, 0xf4, 0x1a, 0xa8, 0xc3, 0xbd, 0x14, 0x75, 0xe8, 0x96, 0x26, 0x91, 0x88, 0xa7, 0x94, 0x35, 0x1d, 0xa2, 0xd3, 0x2b, 0xa0, 0x0e, 0x7b, 0x31, 0xea, 0xd0, 0xc8, 0x4d, 0x22, 0xb1, 0x9f, 0x5c,
0xec, 0x67, 0x58, 0xc3, 0xcb, 0x97, 0xe0, 0x5c, 0x4c, 0x18, 0xde, 0x87, 0x66, 0xe4, 0x2e, 0xd5, 0xd6, 0xb0, 0x9f, 0x60, 0x0d, 0x37, 0x2f, 0xc1, 0xb9, 0x98, 0x30, 0xbc, 0x07, 0x0b, 0x81, 0xbb,
0x25, 0x95, 0xcb, 0xba, 0x64, 0xf7, 0x77, 0x35, 0x58, 0x4c, 0xb8, 0xb8, 0x9a, 0x35, 0x3f, 0xee, 0x58, 0x97, 0x54, 0x2e, 0xeb, 0x92, 0x8d, 0xdf, 0x4c, 0xc3, 0x62, 0xc4, 0xc5, 0x64, 0xd6, 0xfc,
0xc4, 0xa0, 0xc1, 0x3b, 0xc9, 0x4e, 0x95, 0x8d, 0xa8, 0xe1, 0x50, 0x11, 0xcc, 0x6f, 0xf1, 0xeb, 0xba, 0x23, 0x83, 0x06, 0xef, 0x24, 0xdb, 0x45, 0x0e, 0xa2, 0xfa, 0x43, 0x85, 0x37, 0xbf, 0x85,
0x9d, 0x9f, 0x35, 0x3e, 0x80, 0x16, 0x23, 0xee, 0x88, 0xb2, 0x50, 0x26, 0x0e, 0xac, 0x19, 0x93, 0xaf, 0x77, 0x7a, 0xd6, 0xb8, 0x0b, 0x15, 0x46, 0x9c, 0x1e, 0x65, 0xbe, 0x4c, 0x5c, 0xd8, 0x42,
0x87, 0x93, 0x94, 0x14, 0x67, 0xb4, 0xd7, 0xef, 0xc2, 0x52, 0xca, 0xd9, 0x95, 0x86, 0xb0, 0x2f, 0x48, 0x1e, 0x4e, 0x62, 0x52, 0x9c, 0xd0, 0x5e, 0xbf, 0x03, 0x4b, 0x31, 0x67, 0x13, 0x0d, 0x61,
0xf9, 0xe1, 0xc4, 0xc9, 0x79, 0x0d, 0xd9, 0xf5, 0x71, 0x2a, 0xbb, 0xb6, 0xca, 0x0f, 0x33, 0x51, 0x5f, 0xf0, 0xcb, 0x09, 0x93, 0xf3, 0x0a, 0xb2, 0xeb, 0x83, 0x58, 0x76, 0x6d, 0xe6, 0x5f, 0x66,
0x32, 0x65, 0x39, 0x86, 0x33, 0x39, 0xf6, 0x7a, 0x25, 0xb4, 0x8b, 0x33, 0xed, 0x9f, 0x35, 0x58, 0xa4, 0x64, 0xf2, 0x72, 0x0c, 0x27, 0x72, 0xec, 0xb5, 0x42, 0x68, 0x17, 0x67, 0xda, 0x3f, 0xa7,
0x4d, 0x68, 0xc7, 0xdc, 0xf4, 0x87, 0xa9, 0x06, 0xbd, 0x95, 0x69, 0xd0, 0xed, 0x22, 0x9b, 0xe7, 0x61, 0x35, 0xa2, 0x1d, 0x72, 0xd3, 0xef, 0xc7, 0x1a, 0xf4, 0x66, 0xa2, 0x41, 0x57, 0xb3, 0x6c,
0x46, 0x4e, 0x8b, 0x09, 0xe3, 0xec, 0xff, 0x22, 0x61, 0xfc, 0x93, 0x02, 0xcb, 0x89, 0xb3, 0xbb, 0x9e, 0x1b, 0x39, 0xcd, 0x26, 0x8c, 0x33, 0xff, 0x8f, 0x84, 0xf1, 0x0f, 0x0a, 0x2c, 0x47, 0xee,
0x06, 0xc6, 0xf8, 0x20, 0xcd, 0x18, 0x5f, 0xae, 0x92, 0x34, 0x25, 0x94, 0xf1, 0xab, 0xf9, 0x54, 0xee, 0x0a, 0x18, 0xe3, 0x83, 0x38, 0x63, 0xbc, 0x59, 0x24, 0x69, 0x72, 0x28, 0xe3, 0xbf, 0x14,
0xf0, 0xff, 0xf7, 0x24, 0xe6, 0xd7, 0xb0, 0x3a, 0xb6, 0x0d, 0xdf, 0xa4, 0x7b, 0x06, 0xd1, 0xcd, 0x68, 0x46, 0xb4, 0x8e, 0xa8, 0xe3, 0x6a, 0x2e, 0xa3, 0x26, 0xfb, 0xd8, 0xd2, 0xfb, 0x06, 0xdd,
0x50, 0x81, 0x0f, 0x7d, 0xb3, 0xd9, 0x3f, 0x86, 0x22, 0x78, 0xea, 0x7a, 0xba, 0xc7, 0xa8, 0xc5, 0xd5, 0x89, 0x66, 0x60, 0xca, 0x17, 0x34, 0xcb, 0x3c, 0xb2, 0x74, 0xad, 0x33, 0x44, 0x04, 0x16,
0x3e, 0x8d, 0x2d, 0xfb, 0xdf, 0x95, 0x4e, 0x56, 0x3f, 0x2d, 0x80, 0xc3, 0x85, 0x4e, 0xd0, 0x0f, 0x3f, 0x3f, 0xa3, 0xe6, 0x1e, 0xd5, 0x29, 0xa3, 0x5d, 0x99, 0x4e, 0x3f, 0x90, 0xf0, 0x8b, 0x0f,
0x60, 0x91, 0x0f, 0xcc, 0xba, 0x46, 0x39, 0xf7, 0x96, 0x89, 0xb5, 0x22, 0x81, 0x16, 0x8f, 0x63, 0x43, 0xd1, 0xb3, 0x51, 0x7d, 0xb3, 0x08, 0xa2, 0xc8, 0xb2, 0x28, 0x26, 0xfa, 0x39, 0x00, 0xff,
0x11, 0x4e, 0xea, 0xa1, 0x47, 0xb0, 0xe2, 0xd8, 0xc3, 0x03, 0x62, 0x91, 0x11, 0xe5, 0x63, 0xc6, 0x14, 0xfd, 0xa8, 0x2b, 0x13, 0xee, 0xae, 0x5f, 0x95, 0x0f, 0x03, 0xc9, 0x44, 0x0e, 0x22, 0x88,
0x91, 0x6d, 0xe8, 0xda, 0x44, 0x30, 0x9b, 0x66, 0xff, 0xdd, 0x70, 0xb8, 0x3c, 0xca, 0xab, 0x3c, 0x8d, 0xbf, 0xcd, 0xc7, 0x62, 0xf6, 0x8d, 0xe7, 0x6e, 0xbf, 0x80, 0xd5, 0x41, 0x78, 0x3b, 0xbe,
0xe3, 0x14, 0x21, 0xbf, 0x2c, 0x8a, 0xba, 0x08, 0x12, 0xb9, 0xd0, 0xf2, 0xe5, 0x73, 0x2f, 0x89, 0x02, 0x9f, 0x75, 0x67, 0x92, 0xff, 0x87, 0x05, 0xf0, 0x59, 0xf7, 0xda, 0xfa, 0xb6, 0x74, 0xb2,
0x5e, 0xf0, 0x17, 0xce, 0x4e, 0x95, 0x0c, 0x3b, 0x4d, 0x59, 0xc6, 0xdd, 0x3f, 0xbd, 0x8e, 0x33, 0xfa, 0x71, 0x06, 0x1c, 0xce, 0x74, 0x82, 0xbe, 0x07, 0x8b, 0x9c, 0x27, 0x68, 0x1d, 0xfa, 0x21,
0x1e, 0x4a, 0x89, 0x5b, 0xe3, 0xbf, 0x22, 0x6e, 0x05, 0x4c, 0xb2, 0x79, 0x35, 0x26, 0xd9, 0xfd, 0x31, 0xfc, 0x7a, 0x5a, 0xf1, 0xf3, 0xe5, 0x38, 0x14, 0xe1, 0xa8, 0x1e, 0x3a, 0x83, 0x15, 0xdb,
0x43, 0x1d, 0x6e, 0xe6, 0xba, 0x2d, 0xfa, 0xf1, 0x05, 0x24, 0xe9, 0xd6, 0x73, 0x23, 0x48, 0x39, 0xea, 0x1e, 0x10, 0x93, 0xf4, 0x28, 0x9f, 0xae, 0xbc, 0x50, 0x0a, 0x42, 0xb7, 0xd0, 0x7a, 0xc7,
0x76, 0x33, 0x7b, 0x05, 0x76, 0xb3, 0x0b, 0xcb, 0x9a, 0xef, 0xba, 0xd4, 0x62, 0x19, 0x6e, 0x13, 0x9f, 0xa9, 0x8f, 0xd2, 0x2a, 0xcf, 0x38, 0x33, 0x4a, 0x2f, 0x8b, 0x24, 0xc8, 0x82, 0x44, 0x0e,
0x9d, 0xc5, 0x5e, 0x5a, 0x8c, 0xb3, 0xfa, 0x45, 0x04, 0xad, 0x7e, 0x45, 0x82, 0x96, 0x8c, 0x42, 0x54, 0xfa, 0x72, 0xca, 0x91, 0xfc, 0xd6, 0xfb, 0xe7, 0x6a, 0xbb, 0x48, 0x61, 0x9d, 0xc6, 0x2c,
0x0e, 0xd9, 0x41, 0xe6, 0xe6, 0xa3, 0x90, 0xb3, 0x76, 0x56, 0x9f, 0x0f, 0x18, 0x01, 0x6a, 0x84, 0xc3, 0x47, 0x2f, 0xbe, 0x8e, 0x13, 0x1e, 0x72, 0xf9, 0x6a, 0xf9, 0x7f, 0xe2, 0xab, 0x19, 0x04,
0xb0, 0x90, 0x1e, 0x30, 0x4e, 0x53, 0x52, 0x9c, 0xd1, 0x2e, 0x20, 0x3b, 0xcd, 0xaa, 0x64, 0x07, 0x7a, 0x61, 0x42, 0x02, 0xfd, 0x67, 0x05, 0x6e, 0xda, 0x05, 0x6a, 0xa9, 0x0a, 0xe2, 0x6e, 0xee,
0x91, 0x14, 0x15, 0x03, 0xd1, 0x26, 0xb6, 0xab, 0x94, 0x43, 0x75, 0x2e, 0x56, 0xc8, 0x42, 0x17, 0x17, 0xb9, 0x9b, 0x22, 0xb5, 0xd9, 0xda, 0x1c, 0x8f, 0xea, 0x37, 0x8b, 0x68, 0xe2, 0x42, 0xfb,
0xaf, 0xce, 0x42, 0xbb, 0x7f, 0x55, 0xe0, 0xc5, 0xd2, 0x82, 0x44, 0xbb, 0xa9, 0xe7, 0x7f, 0x3b, 0x6b, 0xfc, 0xae, 0x04, 0xd7, 0x53, 0xaf, 0x25, 0xfa, 0xd1, 0x05, 0x24, 0xf7, 0xc6, 0x73, 0x23,
0xf3, 0xfc, 0x7f, 0xaf, 0xd4, 0x30, 0x31, 0x03, 0xb8, 0xc5, 0x34, 0xed, 0xfd, 0x6a, 0x34, 0xad, 0xb8, 0x29, 0x76, 0x3a, 0x33, 0x01, 0x3b, 0xdd, 0x81, 0xe5, 0x4e, 0xdf, 0x71, 0xa8, 0xc9, 0x12,
0x80, 0x43, 0x5c, 0xce, 0xd7, 0xfa, 0xdb, 0x4f, 0x9e, 0x76, 0x66, 0xbe, 0x7e, 0xda, 0x99, 0xf9, 0xdc, 0x34, 0x08, 0xea, 0x6e, 0x5c, 0x8c, 0x93, 0xfa, 0x59, 0x04, 0xbb, 0x34, 0x21, 0xc1, 0x8e,
0xe6, 0x69, 0x67, 0xe6, 0x37, 0xd3, 0x8e, 0xf2, 0x64, 0xda, 0x51, 0xbe, 0x9e, 0x76, 0x94, 0x6f, 0xee, 0x42, 0x92, 0x24, 0xaf, 0x04, 0xd3, 0xbb, 0x90, 0x5c, 0x29, 0xa9, 0xcf, 0x07, 0x44, 0x0f,
0xa6, 0x1d, 0xe5, 0xef, 0xd3, 0x8e, 0xf2, 0xd5, 0xb7, 0x9d, 0x99, 0xcf, 0x16, 0xa4, 0xc7, 0xff, 0x35, 0x40, 0x98, 0x8f, 0x0f, 0x88, 0xa7, 0x31, 0x29, 0x4e, 0x68, 0x67, 0x90, 0xd5, 0x85, 0xa2,
0x04, 0x00, 0x00, 0xff, 0xff, 0xbd, 0x02, 0xfa, 0x27, 0xed, 0x1b, 0x00, 0x00, 0x64, 0x15, 0x91, 0x18, 0x95, 0x06, 0xd1, 0xef, 0xb6, 0x8a, 0xe4, 0x6e, 0x71, 0x2e, 0x9d, 0xf9,
0x2f, 0xc2, 0xe2, 0xe4, 0xff, 0x22, 0x34, 0xfe, 0xa2, 0xc0, 0x8b, 0xb9, 0x9d, 0x05, 0xed, 0xc4,
0xc6, 0xb7, 0xad, 0xc4, 0xf8, 0xf6, 0x9d, 0x5c, 0xc3, 0xc8, 0x0c, 0xe7, 0x64, 0xd3, 0xec, 0xf7,
0x8a, 0xd1, 0xec, 0x0c, 0x0e, 0x78, 0x39, 0xdf, 0x6e, 0x6d, 0x3d, 0x79, 0x5a, 0x9b, 0xfa, 0xf2,
0x69, 0x6d, 0xea, 0xab, 0xa7, 0xb5, 0xa9, 0x5f, 0x8e, 0x6b, 0xca, 0x93, 0x71, 0x4d, 0xf9, 0x72,
0x5c, 0x53, 0xbe, 0x1a, 0xd7, 0x94, 0xbf, 0x8f, 0x6b, 0xca, 0xaf, 0xbf, 0xae, 0x4d, 0x7d, 0x32,
0x2f, 0x3d, 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0x3d, 0xfc, 0xe0, 0xc3, 0xad, 0x1d, 0x00, 0x00,
} }
func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { func (m *ControllerRevision) Marshal() (dAtA []byte, err error) {
@ -1687,6 +1724,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
i -= len(m.WhenScaled)
copy(dAtA[i:], m.WhenScaled)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled)))
i--
dAtA[i] = 0x12
i -= len(m.WhenDeleted)
copy(dAtA[i:], m.WhenDeleted)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted)))
i--
dAtA[i] = 0xa
return len(dAtA) - i, nil
}
func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) {
size := m.Size() size := m.Size()
dAtA = make([]byte, size) dAtA = make([]byte, size)
@ -1707,6 +1777,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if m.PersistentVolumeClaimRetentionPolicy != nil {
{
size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintGenerated(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x52
}
i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds))
i-- i--
dAtA[i] = 0x48 dAtA[i] = 0x48
@ -2213,6 +2295,19 @@ func (m *StatefulSetList) Size() (n int) {
return n return n
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.WhenDeleted)
n += 1 + l + sovGenerated(uint64(l))
l = len(m.WhenScaled)
n += 1 + l + sovGenerated(uint64(l))
return n
}
func (m *StatefulSetSpec) Size() (n int) { func (m *StatefulSetSpec) Size() (n int) {
if m == nil { if m == nil {
return 0 return 0
@ -2244,6 +2339,10 @@ func (m *StatefulSetSpec) Size() (n int) {
n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit))
} }
n += 1 + sovGenerated(uint64(m.MinReadySeconds)) n += 1 + sovGenerated(uint64(m.MinReadySeconds))
if m.PersistentVolumeClaimRetentionPolicy != nil {
l = m.PersistentVolumeClaimRetentionPolicy.Size()
n += 1 + l + sovGenerated(uint64(l))
}
return n return n
} }
@ -2559,6 +2658,17 @@ func (this *StatefulSetList) String() string {
}, "") }, "")
return s return s
} }
func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`,
`WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`,
`WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`,
`}`,
}, "")
return s
}
func (this *StatefulSetSpec) String() string { func (this *StatefulSetSpec) String() string {
if this == nil { if this == nil {
return "nil" return "nil"
@ -2578,6 +2688,7 @@ func (this *StatefulSetSpec) String() string {
`UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`,
`RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`,
`MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`,
`PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`,
`}`, `}`,
}, "") }, "")
return s return s
@ -5436,6 +5547,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthGenerated
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0
@ -5724,6 +5949,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
break break
} }
} }
case 10:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.PersistentVolumeClaimRetentionPolicy == nil {
m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:]) skippy, err := skipGenerated(dAtA[iNdEx:])

View File

@ -367,6 +367,23 @@ message StatefulSetList {
repeated StatefulSet items = 2; repeated StatefulSet items = 2;
} }
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
message StatefulSetPersistentVolumeClaimRetentionPolicy {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
optional string whenDeleted = 1;
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
optional string whenScaled = 2;
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
message StatefulSetSpec { message StatefulSetSpec {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -434,6 +451,12 @@ message StatefulSetSpec {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
optional int32 minReadySeconds = 9; optional int32 minReadySeconds = 9;
// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10;
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -158,6 +158,40 @@ type RollingUpdateStatefulSetStrategy struct {
Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"`
} }
// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
// deleted or scaled down.
type PersistentVolumeClaimRetentionPolicyType string
const (
// RetainPersistentVolumeClaimRetentionPolicyType is the default
// PersistentVolumeClaimRetentionPolicy and specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will not be deleted.
RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
// RetentionPersistentVolumeClaimRetentionPolicyType specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will be deleted in the scenario specified in
// StatefulSetPersistentVolumeClaimRetentionPolicy.
RetentionPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
)
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"`
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"`
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
type StatefulSetSpec struct { type StatefulSetSpec struct {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -225,6 +259,12 @@ type StatefulSetSpec struct {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"`
// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"`
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -227,17 +227,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string {
return map_StatefulSetList return map_StatefulSetList
} }
var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{
"": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
}
func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string {
return map_StatefulSetPersistentVolumeClaimRetentionPolicy
}
var map_StatefulSetSpec = map[string]string{ var map_StatefulSetSpec = map[string]string{
"": "A StatefulSetSpec is the specification of a StatefulSet.", "": "A StatefulSetSpec is the specification of a StatefulSet.",
"replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.",
"selector": "selector is a label query over pods that should match the replica count. If empty, defaulted to labels on the pod template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", "selector": "selector is a label query over pods that should match the replica count. If empty, defaulted to labels on the pod template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors",
"template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.",
"volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.",
"serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.",
"podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.",
"updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.",
"revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.",
"minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.",
"persistentVolumeClaimRetentionPolicy": "PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.",
} }
func (StatefulSetSpec) SwaggerDoc() map[string]string { func (StatefulSetSpec) SwaggerDoc() map[string]string {

View File

@ -500,6 +500,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object {
return nil return nil
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy {
if in == nil {
return nil
}
out := new(StatefulSetPersistentVolumeClaimRetentionPolicy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = *in *out = *in
@ -527,6 +543,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = new(int32) *out = new(int32)
**out = **in **out = **in
} }
if in.PersistentVolumeClaimRetentionPolicy != nil {
in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy
*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
**out = **in
}
return return
} }

View File

@ -833,10 +833,40 @@ func (m *StatefulSetList) XXX_DiscardUnknown() {
var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo var xxx_messageInfo_StatefulSetList proto.InternalMessageInfo
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Reset() {
*m = StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) ProtoMessage() {}
func (*StatefulSetPersistentVolumeClaimRetentionPolicy) Descriptor() ([]byte, []int) {
return fileDescriptor_42fe616264472f7e, []int{28}
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Merge(src proto.Message) {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.Merge(m, src)
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_Size() int {
return m.Size()
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) XXX_DiscardUnknown() {
xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy.DiscardUnknown(m)
}
var xxx_messageInfo_StatefulSetPersistentVolumeClaimRetentionPolicy proto.InternalMessageInfo
func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} } func (m *StatefulSetSpec) Reset() { *m = StatefulSetSpec{} }
func (*StatefulSetSpec) ProtoMessage() {} func (*StatefulSetSpec) ProtoMessage() {}
func (*StatefulSetSpec) Descriptor() ([]byte, []int) { func (*StatefulSetSpec) Descriptor() ([]byte, []int) {
return fileDescriptor_42fe616264472f7e, []int{28} return fileDescriptor_42fe616264472f7e, []int{29}
} }
func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error { func (m *StatefulSetSpec) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -864,7 +894,7 @@ var xxx_messageInfo_StatefulSetSpec proto.InternalMessageInfo
func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} } func (m *StatefulSetStatus) Reset() { *m = StatefulSetStatus{} }
func (*StatefulSetStatus) ProtoMessage() {} func (*StatefulSetStatus) ProtoMessage() {}
func (*StatefulSetStatus) Descriptor() ([]byte, []int) { func (*StatefulSetStatus) Descriptor() ([]byte, []int) {
return fileDescriptor_42fe616264472f7e, []int{29} return fileDescriptor_42fe616264472f7e, []int{30}
} }
func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error { func (m *StatefulSetStatus) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -892,7 +922,7 @@ var xxx_messageInfo_StatefulSetStatus proto.InternalMessageInfo
func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} } func (m *StatefulSetUpdateStrategy) Reset() { *m = StatefulSetUpdateStrategy{} }
func (*StatefulSetUpdateStrategy) ProtoMessage() {} func (*StatefulSetUpdateStrategy) ProtoMessage() {}
func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) { func (*StatefulSetUpdateStrategy) Descriptor() ([]byte, []int) {
return fileDescriptor_42fe616264472f7e, []int{30} return fileDescriptor_42fe616264472f7e, []int{31}
} }
func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error { func (m *StatefulSetUpdateStrategy) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b) return m.Unmarshal(b)
@ -947,6 +977,7 @@ func init() {
proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta2.StatefulSet") proto.RegisterType((*StatefulSet)(nil), "k8s.io.api.apps.v1beta2.StatefulSet")
proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta2.StatefulSetCondition") proto.RegisterType((*StatefulSetCondition)(nil), "k8s.io.api.apps.v1beta2.StatefulSetCondition")
proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta2.StatefulSetList") proto.RegisterType((*StatefulSetList)(nil), "k8s.io.api.apps.v1beta2.StatefulSetList")
proto.RegisterType((*StatefulSetPersistentVolumeClaimRetentionPolicy)(nil), "k8s.io.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy")
proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta2.StatefulSetSpec") proto.RegisterType((*StatefulSetSpec)(nil), "k8s.io.api.apps.v1beta2.StatefulSetSpec")
proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta2.StatefulSetStatus") proto.RegisterType((*StatefulSetStatus)(nil), "k8s.io.api.apps.v1beta2.StatefulSetStatus")
proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta2.StatefulSetUpdateStrategy") proto.RegisterType((*StatefulSetUpdateStrategy)(nil), "k8s.io.api.apps.v1beta2.StatefulSetUpdateStrategy")
@ -957,144 +988,150 @@ func init() {
} }
var fileDescriptor_42fe616264472f7e = []byte{ var fileDescriptor_42fe616264472f7e = []byte{
// 2182 bytes of a gzipped FileDescriptorProto // 2284 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x1c, 0xb7, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcf, 0x6f, 0x1b, 0xc7,
0xf9, 0xf6, 0xec, 0x87, 0xb4, 0x4b, 0x59, 0x92, 0x4d, 0xe9, 0x27, 0x6d, 0xec, 0x5f, 0x57, 0xc6, 0xf5, 0xf7, 0xf2, 0x87, 0x44, 0x8e, 0x2c, 0xc9, 0x1e, 0xe9, 0x2b, 0x31, 0xf2, 0xb7, 0xa4, 0xb1,
0x26, 0x70, 0x94, 0xd8, 0x9a, 0xb5, 0x95, 0x0f, 0x24, 0x76, 0xd1, 0x56, 0x2b, 0xa5, 0xb6, 0x03, 0x31, 0x1c, 0x25, 0xb6, 0x48, 0x5b, 0xf9, 0x81, 0xc4, 0x2e, 0x92, 0x8a, 0x52, 0x6a, 0x3b, 0xd0,
0x7d, 0x85, 0xb2, 0x0c, 0x34, 0x68, 0x51, 0x53, 0xbb, 0xf4, 0x6a, 0xa2, 0xf9, 0xc2, 0x0c, 0x67, 0x0f, 0x66, 0x64, 0x39, 0x68, 0xd0, 0x1f, 0x1e, 0x91, 0x63, 0x6a, 0xa3, 0xe5, 0xee, 0x62, 0x77,
0xeb, 0x45, 0x2f, 0xbd, 0x16, 0x28, 0xd0, 0xf4, 0xda, 0x7f, 0xa2, 0xb7, 0xa2, 0x68, 0x6e, 0x45, 0x96, 0x31, 0xd1, 0x4b, 0xaf, 0x05, 0x0a, 0xb4, 0xbd, 0xf6, 0x9f, 0xe8, 0xad, 0x28, 0x9a, 0x5b,
0x50, 0xf8, 0x18, 0xf4, 0x92, 0x9c, 0x84, 0x7a, 0x73, 0x2a, 0x8a, 0x1e, 0x7b, 0x09, 0x50, 0xa0, 0x11, 0x04, 0x3e, 0x06, 0xbd, 0x24, 0xbd, 0x10, 0x35, 0x73, 0x2a, 0x8a, 0xde, 0xda, 0x8b, 0x81,
0x20, 0x87, 0xf3, 0xc1, 0xf9, 0xf0, 0x8e, 0x14, 0x47, 0x69, 0x82, 0xdc, 0xb4, 0xe4, 0xf3, 0x3e, 0x02, 0xc5, 0xcc, 0xce, 0xfe, 0xde, 0x35, 0x97, 0x8a, 0xa3, 0x34, 0x41, 0x6e, 0xe2, 0xbc, 0xf7,
0x7c, 0x5f, 0xf2, 0x25, 0xdf, 0x87, 0x1c, 0x81, 0x1f, 0x1d, 0xbd, 0xe5, 0xaa, 0x9a, 0xd5, 0x3e, 0x3e, 0xf3, 0xde, 0xcc, 0x7b, 0xf3, 0x3e, 0x33, 0x2b, 0xf0, 0x83, 0xe3, 0xd7, 0xad, 0xba, 0xa2,
0xf2, 0x0e, 0x88, 0x63, 0x12, 0x4a, 0xdc, 0xf6, 0x80, 0x98, 0x3d, 0xcb, 0x69, 0x8b, 0x0e, 0x6c, 0x37, 0x8e, 0xed, 0x43, 0x62, 0x6a, 0x84, 0x12, 0xab, 0xd1, 0x27, 0x5a, 0x47, 0x37, 0x1b, 0x42,
0x6b, 0x6d, 0x6c, 0xdb, 0x6e, 0x7b, 0x70, 0xf3, 0x80, 0x50, 0xbc, 0xda, 0xee, 0x13, 0x93, 0x38, 0x80, 0x0d, 0xa5, 0x81, 0x0d, 0xc3, 0x6a, 0xf4, 0xaf, 0x1f, 0x12, 0x8a, 0xd7, 0x1b, 0x5d, 0xa2,
0x98, 0x92, 0x9e, 0x6a, 0x3b, 0x16, 0xb5, 0xe0, 0xa2, 0x0f, 0x54, 0xb1, 0xad, 0xa9, 0x0c, 0xa8, 0x11, 0x13, 0x53, 0xd2, 0xa9, 0x1b, 0xa6, 0x4e, 0x75, 0xb8, 0xec, 0x28, 0xd6, 0xb1, 0xa1, 0xd4,
0x0a, 0xe0, 0xa5, 0x95, 0xbe, 0x46, 0x0f, 0xbd, 0x03, 0xb5, 0x6b, 0x19, 0xed, 0xbe, 0xd5, 0xb7, 0x99, 0x62, 0x5d, 0x28, 0xae, 0xac, 0x75, 0x15, 0x7a, 0x64, 0x1f, 0xd6, 0xdb, 0x7a, 0xaf, 0xd1,
0xda, 0x1c, 0x7f, 0xe0, 0x3d, 0xe2, 0xbf, 0xf8, 0x0f, 0xfe, 0x97, 0xcf, 0x73, 0xa9, 0x15, 0x1b, 0xd5, 0xbb, 0x7a, 0x83, 0xeb, 0x1f, 0xda, 0x0f, 0xf8, 0x2f, 0xfe, 0x83, 0xff, 0xe5, 0xe0, 0xac,
0xb0, 0x6b, 0x39, 0xa4, 0x3d, 0xb8, 0x99, 0x1c, 0xeb, 0xd2, 0xeb, 0x11, 0xc6, 0xc0, 0xdd, 0x43, 0xc8, 0x81, 0x09, 0xdb, 0xba, 0x49, 0x1a, 0xfd, 0xeb, 0xd1, 0xb9, 0x56, 0x5e, 0xf1, 0x75, 0x7a,
0xcd, 0x24, 0xce, 0xb0, 0x6d, 0x1f, 0xf5, 0x59, 0x83, 0xdb, 0x36, 0x08, 0xc5, 0x59, 0x56, 0xed, 0xb8, 0x7d, 0xa4, 0x68, 0xc4, 0x1c, 0x34, 0x8c, 0xe3, 0x2e, 0x1b, 0xb0, 0x1a, 0x3d, 0x42, 0x71,
0x3c, 0x2b, 0xc7, 0x33, 0xa9, 0x66, 0x90, 0x94, 0xc1, 0x9b, 0xe3, 0x0c, 0xdc, 0xee, 0x21, 0x31, 0x92, 0x55, 0x23, 0xcd, 0xca, 0xb4, 0x35, 0xaa, 0xf4, 0x48, 0xcc, 0xe0, 0xb5, 0x71, 0x06, 0x56,
0x70, 0xca, 0xee, 0xb5, 0x3c, 0x3b, 0x8f, 0x6a, 0x7a, 0x5b, 0x33, 0xa9, 0x4b, 0x9d, 0xa4, 0x51, 0xfb, 0x88, 0xf4, 0x70, 0xcc, 0xee, 0xe5, 0x34, 0x3b, 0x9b, 0x2a, 0x6a, 0x43, 0xd1, 0xa8, 0x45,
0xeb, 0xdf, 0x0a, 0x80, 0xeb, 0x96, 0x49, 0x1d, 0x4b, 0xd7, 0x89, 0x83, 0xc8, 0x40, 0x73, 0x35, 0xcd, 0xa8, 0x91, 0xfc, 0x6f, 0x09, 0xc0, 0x4d, 0x5d, 0xa3, 0xa6, 0xae, 0xaa, 0xc4, 0x44, 0xa4,
0xcb, 0x84, 0x0f, 0x41, 0x8d, 0xc5, 0xd3, 0xc3, 0x14, 0x37, 0x94, 0x2b, 0xca, 0xf2, 0xd4, 0xea, 0xaf, 0x58, 0x8a, 0xae, 0xc1, 0xfb, 0xa0, 0xc4, 0xe2, 0xe9, 0x60, 0x8a, 0x2b, 0xd2, 0x45, 0x69,
0x0d, 0x35, 0x9a, 0xe9, 0x90, 0x5e, 0xb5, 0x8f, 0xfa, 0xac, 0xc1, 0x55, 0x19, 0x5a, 0x1d, 0xdc, 0x75, 0x66, 0xfd, 0x5a, 0xdd, 0x5f, 0x69, 0x0f, 0xbe, 0x6e, 0x1c, 0x77, 0xd9, 0x80, 0x55, 0x67,
0x54, 0x77, 0x0e, 0x3e, 0x20, 0x5d, 0xba, 0x45, 0x28, 0xee, 0xc0, 0x27, 0xc7, 0x4b, 0xe7, 0x46, 0xda, 0xf5, 0xfe, 0xf5, 0xfa, 0xde, 0xe1, 0x07, 0xa4, 0x4d, 0x77, 0x08, 0xc5, 0x4d, 0xf8, 0x68,
0xc7, 0x4b, 0x20, 0x6a, 0x43, 0x21, 0x2b, 0xdc, 0x01, 0x15, 0xce, 0x5e, 0xe2, 0xec, 0x2b, 0xb9, 0x58, 0x3b, 0x33, 0x1a, 0xd6, 0x80, 0x3f, 0x86, 0x3c, 0x54, 0xb8, 0x07, 0x0a, 0x1c, 0x3d, 0xc7,
0xec, 0x22, 0x68, 0x15, 0xe1, 0x5f, 0xbc, 0xf3, 0x98, 0x12, 0x93, 0xb9, 0xd7, 0x39, 0x2f, 0xa8, 0xd1, 0xd7, 0x52, 0xd1, 0x45, 0xd0, 0x75, 0x84, 0x3f, 0x7c, 0xfb, 0x21, 0x25, 0x1a, 0x73, 0xaf,
0x2b, 0x1b, 0x98, 0x62, 0xc4, 0x89, 0xe0, 0x75, 0x50, 0x73, 0x84, 0xfb, 0x8d, 0xf2, 0x15, 0x65, 0x79, 0x56, 0x40, 0x17, 0xb6, 0x30, 0xc5, 0x88, 0x03, 0xc1, 0xab, 0xa0, 0x64, 0x0a, 0xf7, 0x2b,
0xb9, 0xdc, 0xb9, 0x20, 0x50, 0xb5, 0x20, 0x2c, 0x14, 0x22, 0x5a, 0x4f, 0x14, 0xb0, 0x90, 0x8e, 0xf9, 0x8b, 0xd2, 0x6a, 0xbe, 0x79, 0x4e, 0x68, 0x95, 0xdc, 0xb0, 0x90, 0xa7, 0x21, 0x3f, 0x92,
0x7b, 0x53, 0x73, 0x29, 0xfc, 0x69, 0x2a, 0x76, 0xb5, 0x58, 0xec, 0xcc, 0x9a, 0x47, 0x1e, 0x0e, 0xc0, 0x52, 0x3c, 0xee, 0x6d, 0xc5, 0xa2, 0xf0, 0xc7, 0xb1, 0xd8, 0xeb, 0xd9, 0x62, 0x67, 0xd6,
0x1c, 0xb4, 0xc4, 0xe2, 0xde, 0x05, 0x55, 0x8d, 0x12, 0xc3, 0x6d, 0x94, 0xae, 0x94, 0x97, 0xa7, 0x3c, 0x72, 0x6f, 0x62, 0x77, 0x24, 0x10, 0x77, 0x0b, 0x14, 0x15, 0x4a, 0x7a, 0x56, 0x25, 0x77,
0x56, 0xaf, 0xa9, 0x39, 0x09, 0xac, 0xa6, 0xbd, 0xeb, 0x4c, 0x0b, 0xde, 0xea, 0x3d, 0xc6, 0x80, 0x31, 0xbf, 0x3a, 0xb3, 0x7e, 0xa5, 0x9e, 0x92, 0xc0, 0xf5, 0xb8, 0x77, 0xcd, 0x59, 0x81, 0x5b,
0x7c, 0xa2, 0xd6, 0xaf, 0x4b, 0xa0, 0xbe, 0x81, 0x89, 0x61, 0x99, 0x7b, 0x84, 0x9e, 0xc1, 0xca, 0xbc, 0xc3, 0x10, 0x90, 0x03, 0x24, 0xff, 0x32, 0x07, 0xca, 0x5b, 0x98, 0xf4, 0x74, 0x6d, 0x9f,
0xdd, 0x05, 0x15, 0xd7, 0x26, 0x5d, 0xb1, 0x72, 0x57, 0x73, 0x03, 0x08, 0x7d, 0xda, 0xb3, 0x49, 0xd0, 0x53, 0xd8, 0xb9, 0xdb, 0xa0, 0x60, 0x19, 0xa4, 0x2d, 0x76, 0xee, 0x72, 0x6a, 0x00, 0x9e,
0x37, 0x5a, 0x32, 0xf6, 0x0b, 0x71, 0x06, 0xb8, 0x0b, 0x26, 0x5c, 0x8a, 0xa9, 0xe7, 0xf2, 0x05, 0x4f, 0xfb, 0x06, 0x69, 0xfb, 0x5b, 0xc6, 0x7e, 0x21, 0x8e, 0x00, 0x5b, 0x60, 0xca, 0xa2, 0x98,
0x9b, 0x5a, 0x5d, 0x2e, 0xc0, 0xc5, 0xf1, 0x9d, 0x19, 0xc1, 0x36, 0xe1, 0xff, 0x46, 0x82, 0xa7, 0xda, 0x16, 0xdf, 0xb0, 0x99, 0xf5, 0xd5, 0x0c, 0x58, 0x5c, 0xbf, 0x39, 0x27, 0xd0, 0xa6, 0x9c,
0xf5, 0x8f, 0x12, 0x80, 0x21, 0x76, 0xdd, 0x32, 0x7b, 0x1a, 0x65, 0xe9, 0x7c, 0x0b, 0x54, 0xe8, 0xdf, 0x48, 0xe0, 0xc8, 0x7f, 0xcf, 0x01, 0xe8, 0xe9, 0x6e, 0xea, 0x5a, 0x47, 0xa1, 0x2c, 0x9d,
0xd0, 0x26, 0x7c, 0x42, 0xea, 0x9d, 0xab, 0x81, 0x2b, 0xf7, 0x87, 0x36, 0xf9, 0xe2, 0x78, 0x69, 0x6f, 0x80, 0x02, 0x1d, 0x18, 0x84, 0x2f, 0x48, 0xb9, 0x79, 0xd9, 0x75, 0xe5, 0xee, 0xc0, 0x20,
0x21, 0x6d, 0xc1, 0x7a, 0x10, 0xb7, 0x81, 0x9b, 0xa1, 0x93, 0x25, 0x6e, 0xfd, 0xba, 0x3c, 0xf4, 0x4f, 0x86, 0xb5, 0xa5, 0xb8, 0x05, 0x93, 0x20, 0x6e, 0x03, 0xb7, 0x3d, 0x27, 0x73, 0xdc, 0xfa,
0x17, 0xc7, 0x4b, 0x19, 0x67, 0x87, 0x1a, 0x32, 0xc9, 0x0e, 0xc2, 0x01, 0x80, 0x3a, 0x76, 0xe9, 0x95, 0xf0, 0xd4, 0x4f, 0x86, 0xb5, 0x84, 0xb3, 0xa3, 0xee, 0x21, 0x85, 0x1d, 0x84, 0x7d, 0x00,
0x7d, 0x07, 0x9b, 0xae, 0x3f, 0x92, 0x66, 0x10, 0x11, 0xfe, 0xab, 0xc5, 0x16, 0x8a, 0x59, 0x74, 0x55, 0x6c, 0xd1, 0xbb, 0x26, 0xd6, 0x2c, 0x67, 0x26, 0xa5, 0x47, 0x44, 0xf8, 0x2f, 0x65, 0xdb,
0x2e, 0x09, 0x2f, 0xe0, 0x66, 0x8a, 0x0d, 0x65, 0x8c, 0x00, 0xaf, 0x82, 0x09, 0x87, 0x60, 0xd7, 0x28, 0x66, 0xd1, 0x5c, 0x11, 0x5e, 0xc0, 0xed, 0x18, 0x1a, 0x4a, 0x98, 0x01, 0x5e, 0x06, 0x53,
0x32, 0x1b, 0x15, 0x1e, 0x45, 0x38, 0x81, 0x88, 0xb7, 0x22, 0xd1, 0x0b, 0x5f, 0x01, 0x93, 0x06, 0x26, 0xc1, 0x96, 0xae, 0x55, 0x0a, 0x3c, 0x0a, 0x6f, 0x01, 0x11, 0x1f, 0x45, 0x42, 0x0a, 0x5f,
0x71, 0x5d, 0xdc, 0x27, 0x8d, 0x2a, 0x07, 0xce, 0x0a, 0xe0, 0xe4, 0x96, 0xdf, 0x8c, 0x82, 0xfe, 0x04, 0xd3, 0x3d, 0x62, 0x59, 0xb8, 0x4b, 0x2a, 0x45, 0xae, 0x38, 0x2f, 0x14, 0xa7, 0x77, 0x9c,
0xd6, 0x1f, 0x15, 0x30, 0x1d, 0xce, 0xdc, 0x19, 0xec, 0x9c, 0x3b, 0xf2, 0xce, 0x69, 0x8d, 0x4f, 0x61, 0xe4, 0xca, 0xe5, 0x3f, 0x48, 0x60, 0xd6, 0x5b, 0xb9, 0x53, 0xa8, 0x9c, 0x5b, 0xe1, 0xca,
0x96, 0x9c, 0x0d, 0xf3, 0x71, 0x39, 0xe6, 0x38, 0x4b, 0x47, 0xf8, 0x33, 0x50, 0x73, 0x89, 0x4e, 0x91, 0xc7, 0x27, 0x4b, 0x4a, 0xc1, 0x7c, 0x9c, 0x0f, 0x38, 0xce, 0xd2, 0x11, 0xfe, 0x04, 0x94,
0xba, 0xd4, 0x72, 0x84, 0xe3, 0xaf, 0x15, 0x74, 0x1c, 0x1f, 0x10, 0x7d, 0x4f, 0x98, 0x76, 0xce, 0x2c, 0xa2, 0x92, 0x36, 0xd5, 0x4d, 0xe1, 0xf8, 0xcb, 0x19, 0x1d, 0xc7, 0x87, 0x44, 0xdd, 0x17,
0x33, 0xcf, 0x83, 0x5f, 0x28, 0xa4, 0x84, 0xef, 0x81, 0x1a, 0x25, 0x86, 0xad, 0x63, 0x4a, 0xc4, 0xa6, 0xcd, 0xb3, 0xcc, 0x73, 0xf7, 0x17, 0xf2, 0x20, 0xe1, 0xbb, 0xa0, 0x44, 0x49, 0xcf, 0x50,
0xae, 0x79, 0x31, 0xee, 0x3c, 0xcb, 0x19, 0x46, 0xb6, 0x6b, 0xf5, 0xee, 0x0b, 0x18, 0xdf, 0x32, 0x31, 0x25, 0xa2, 0x6a, 0x9e, 0x0f, 0x3a, 0xcf, 0x72, 0x86, 0x81, 0xb5, 0xf4, 0xce, 0x5d, 0xa1,
0xe1, 0x64, 0x04, 0xad, 0x28, 0xa4, 0x81, 0x36, 0x98, 0xf1, 0xec, 0x1e, 0x43, 0x52, 0x76, 0x9c, 0xc6, 0x4b, 0xc6, 0x5b, 0x0c, 0x77, 0x14, 0x79, 0x30, 0xd0, 0x00, 0x73, 0xb6, 0xd1, 0x61, 0x9a,
0xf7, 0x87, 0x22, 0x87, 0x6e, 0x8c, 0x9f, 0x95, 0x7d, 0xc9, 0xae, 0xb3, 0x20, 0x46, 0x99, 0x91, 0x94, 0x1d, 0xe7, 0xdd, 0x81, 0xc8, 0xa1, 0x6b, 0xe3, 0x57, 0xe5, 0x20, 0x64, 0xd7, 0x5c, 0x12,
0xdb, 0x51, 0x82, 0x1f, 0xae, 0x81, 0x59, 0x43, 0x33, 0x11, 0xc1, 0xbd, 0xe1, 0x1e, 0xe9, 0x5a, 0xb3, 0xcc, 0x85, 0xc7, 0x51, 0x04, 0x1f, 0x6e, 0x80, 0xf9, 0x9e, 0xa2, 0x21, 0x82, 0x3b, 0x83,
0x66, 0xcf, 0xe5, 0xa9, 0x54, 0xed, 0x2c, 0x0a, 0x82, 0xd9, 0x2d, 0xb9, 0x1b, 0x25, 0xf1, 0x70, 0x7d, 0xd2, 0xd6, 0xb5, 0x8e, 0xc5, 0x53, 0xa9, 0xd8, 0x5c, 0x16, 0x00, 0xf3, 0x3b, 0x61, 0x31,
0x13, 0xcc, 0x07, 0x07, 0xf0, 0x5d, 0xcd, 0xa5, 0x96, 0x33, 0xdc, 0xd4, 0x0c, 0x8d, 0x36, 0x26, 0x8a, 0xea, 0xc3, 0x6d, 0xb0, 0xe8, 0x1e, 0xc0, 0xb7, 0x15, 0x8b, 0xea, 0xe6, 0x60, 0x5b, 0xe9,
0x38, 0x4f, 0x63, 0x74, 0xbc, 0x34, 0x8f, 0x32, 0xfa, 0x51, 0xa6, 0x55, 0xeb, 0x77, 0x13, 0x60, 0x29, 0xb4, 0x32, 0xc5, 0x71, 0x2a, 0xa3, 0x61, 0x6d, 0x11, 0x25, 0xc8, 0x51, 0xa2, 0x95, 0xfc,
0x36, 0x71, 0x2e, 0xc0, 0x07, 0x60, 0xa1, 0xeb, 0x39, 0x0e, 0x31, 0xe9, 0xb6, 0x67, 0x1c, 0x10, 0xdb, 0x29, 0x30, 0x1f, 0x39, 0x17, 0xe0, 0x3d, 0xb0, 0xd4, 0xb6, 0x4d, 0x93, 0x68, 0x74, 0xd7,
0x67, 0xaf, 0x7b, 0x48, 0x7a, 0x9e, 0x4e, 0x7a, 0x7c, 0x59, 0xab, 0x9d, 0xa6, 0xf0, 0x75, 0x61, 0xee, 0x1d, 0x12, 0x73, 0xbf, 0x7d, 0x44, 0x3a, 0xb6, 0x4a, 0x3a, 0x7c, 0x5b, 0x8b, 0xcd, 0xaa,
0x3d, 0x13, 0x85, 0x72, 0xac, 0xe1, 0xbb, 0x00, 0x9a, 0xbc, 0x69, 0x4b, 0x73, 0xdd, 0x90, 0xb3, 0xf0, 0x75, 0x69, 0x33, 0x51, 0x0b, 0xa5, 0x58, 0xc3, 0x77, 0x00, 0xd4, 0xf8, 0xd0, 0x8e, 0x62,
0xc4, 0x39, 0xc3, 0xad, 0xb8, 0x9d, 0x42, 0xa0, 0x0c, 0x2b, 0xe6, 0x63, 0x8f, 0xb8, 0x9a, 0x43, 0x59, 0x1e, 0x66, 0x8e, 0x63, 0x7a, 0xa5, 0xb8, 0x1b, 0xd3, 0x40, 0x09, 0x56, 0xcc, 0xc7, 0x0e,
0x7a, 0x49, 0x1f, 0xcb, 0xb2, 0x8f, 0x1b, 0x99, 0x28, 0x94, 0x63, 0x0d, 0xdf, 0x00, 0x53, 0xfe, 0xb1, 0x14, 0x93, 0x74, 0xa2, 0x3e, 0xe6, 0xc3, 0x3e, 0x6e, 0x25, 0x6a, 0xa1, 0x14, 0x6b, 0xf8,
0x68, 0x7c, 0xce, 0xc5, 0xe2, 0xcc, 0x09, 0xb2, 0xa9, 0xed, 0xa8, 0x0b, 0xc5, 0x71, 0x2c, 0x34, 0x2a, 0x98, 0x71, 0x66, 0xe3, 0x6b, 0x2e, 0x36, 0x67, 0x41, 0x80, 0xcd, 0xec, 0xfa, 0x22, 0x14,
0xeb, 0xc0, 0x25, 0xce, 0x80, 0xf4, 0xee, 0xf8, 0xe2, 0x80, 0x55, 0xd0, 0x2a, 0xaf, 0xa0, 0x61, 0xd4, 0x63, 0xa1, 0xe9, 0x87, 0x16, 0x31, 0xfb, 0xa4, 0x73, 0xcb, 0x21, 0x07, 0xac, 0x83, 0x16,
0x68, 0x3b, 0x29, 0x04, 0xca, 0xb0, 0x62, 0xa1, 0xf9, 0x59, 0x93, 0x0a, 0x6d, 0x42, 0x0e, 0x6d, 0x79, 0x07, 0xf5, 0x42, 0xdb, 0x8b, 0x69, 0xa0, 0x04, 0x2b, 0x16, 0x9a, 0x93, 0x35, 0xb1, 0xd0,
0x3f, 0x13, 0x85, 0x72, 0xac, 0x59, 0xee, 0xf9, 0x2e, 0xaf, 0x0d, 0xb0, 0xa6, 0xe3, 0x03, 0x9d, 0xa6, 0xc2, 0xa1, 0x1d, 0x24, 0x6a, 0xa1, 0x14, 0x6b, 0x96, 0x7b, 0x8e, 0xcb, 0x1b, 0x7d, 0xac,
0x34, 0x26, 0xe5, 0xdc, 0xdb, 0x96, 0xbb, 0x51, 0x12, 0x0f, 0xef, 0x80, 0x8b, 0x7e, 0xd3, 0xbe, 0xa8, 0xf8, 0x50, 0x25, 0x95, 0xe9, 0x70, 0xee, 0xed, 0x86, 0xc5, 0x28, 0xaa, 0x0f, 0x6f, 0x81,
0x89, 0x43, 0x92, 0x1a, 0x27, 0x79, 0x41, 0x90, 0x5c, 0xdc, 0x4e, 0x02, 0x50, 0xda, 0x06, 0xde, 0xf3, 0xce, 0xd0, 0x81, 0x86, 0x3d, 0x90, 0x12, 0x07, 0x79, 0x4e, 0x80, 0x9c, 0xdf, 0x8d, 0x2a,
0x02, 0x33, 0x5d, 0x4b, 0xd7, 0x79, 0x3e, 0xae, 0x5b, 0x9e, 0x49, 0x1b, 0x75, 0xce, 0x02, 0xd9, 0xa0, 0xb8, 0x0d, 0xbc, 0x01, 0xe6, 0xda, 0xba, 0xaa, 0xf2, 0x7c, 0xdc, 0xd4, 0x6d, 0x8d, 0x56,
0x1e, 0x5a, 0x97, 0x7a, 0x50, 0x02, 0x09, 0x7f, 0x0e, 0x40, 0x37, 0x28, 0x0c, 0x6e, 0x03, 0x8c, 0xca, 0x1c, 0x05, 0xb2, 0x1a, 0xda, 0x0c, 0x49, 0x50, 0x44, 0x13, 0xfe, 0x0c, 0x80, 0xb6, 0xdb,
0x51, 0x00, 0xe9, 0xb2, 0x14, 0x55, 0xe6, 0xb0, 0xc9, 0x45, 0x31, 0xca, 0xd6, 0xc7, 0x0a, 0x58, 0x18, 0xac, 0x0a, 0x18, 0xc3, 0x00, 0xe2, 0x6d, 0xc9, 0xef, 0xcc, 0xde, 0x90, 0x85, 0x02, 0x90,
0xcc, 0xd9, 0xe8, 0xf0, 0x87, 0x52, 0x11, 0xbc, 0x96, 0x28, 0x82, 0x97, 0x73, 0xcc, 0x62, 0x95, 0xf2, 0xc7, 0x12, 0x58, 0x4e, 0x29, 0x74, 0xf8, 0x56, 0xa8, 0x09, 0x5e, 0x89, 0x34, 0xc1, 0x0b,
0xf0, 0x10, 0x4c, 0x33, 0x41, 0xa2, 0x99, 0x7d, 0x1f, 0x22, 0xce, 0xb2, 0x76, 0x6e, 0x00, 0x28, 0x29, 0x66, 0x81, 0x4e, 0x78, 0x04, 0x66, 0x19, 0x21, 0x51, 0xb4, 0xae, 0xa3, 0x22, 0xce, 0xb2,
0x8e, 0x8e, 0x4e, 0xe5, 0x8b, 0xa3, 0xe3, 0xa5, 0x69, 0xa9, 0x0f, 0xc9, 0xc4, 0xad, 0xdf, 0x94, 0x46, 0x6a, 0x00, 0x28, 0xa8, 0xed, 0x9f, 0xca, 0xe7, 0x47, 0xc3, 0xda, 0x6c, 0x48, 0x86, 0xc2,
0x00, 0xd8, 0x20, 0xb6, 0x6e, 0x0d, 0x0d, 0x62, 0x9e, 0x85, 0xa6, 0xb9, 0x27, 0x69, 0x9a, 0x97, 0xc0, 0xf2, 0xaf, 0x72, 0x00, 0x6c, 0x11, 0x43, 0xd5, 0x07, 0x3d, 0xa2, 0x9d, 0x06, 0xa7, 0xb9,
0xf3, 0x97, 0x24, 0x74, 0x2a, 0x57, 0xd4, 0xbc, 0x97, 0x10, 0x35, 0xaf, 0x14, 0x21, 0x7b, 0xb6, 0x13, 0xe2, 0x34, 0x2f, 0xa4, 0x6f, 0x89, 0xe7, 0x54, 0x2a, 0xa9, 0x79, 0x37, 0x42, 0x6a, 0x5e,
0xaa, 0xf9, 0xb4, 0x0c, 0xe6, 0x22, 0x70, 0x24, 0x6b, 0x6e, 0x4b, 0x2b, 0xfa, 0x72, 0x62, 0x45, 0xcc, 0x02, 0xf6, 0x74, 0x56, 0xf3, 0x59, 0x1e, 0x2c, 0xf8, 0xca, 0x3e, 0xad, 0xb9, 0x19, 0xda,
0x17, 0x33, 0x4c, 0xbe, 0x32, 0x5d, 0xf3, 0x01, 0x98, 0x61, 0xaa, 0xc3, 0x5f, 0x3f, 0xae, 0x69, 0xd1, 0x17, 0x22, 0x3b, 0xba, 0x9c, 0x60, 0xf2, 0x95, 0xf1, 0x9a, 0x0f, 0xc0, 0x1c, 0x63, 0x1d,
0x26, 0x4e, 0xac, 0x69, 0xc2, 0x4a, 0xb4, 0x29, 0x31, 0xa1, 0x04, 0x73, 0x8e, 0x86, 0x9a, 0xfc, 0xce, 0xfe, 0x71, 0x4e, 0x33, 0x35, 0x31, 0xa7, 0xf1, 0x3a, 0xd1, 0x76, 0x08, 0x09, 0x45, 0x90,
0x26, 0x6a, 0xa8, 0x3f, 0x29, 0x60, 0x26, 0x5a, 0xa6, 0x33, 0x10, 0x51, 0x77, 0x65, 0x11, 0xf5, 0x53, 0x38, 0xd4, 0xf4, 0x37, 0x91, 0x43, 0xfd, 0x51, 0x02, 0x73, 0xfe, 0x36, 0x9d, 0x02, 0x89,
0x62, 0x81, 0xe4, 0xcc, 0x51, 0x51, 0x9f, 0x56, 0xe2, 0xae, 0x73, 0x19, 0xb5, 0xcc, 0xae, 0x60, 0xba, 0x1d, 0x26, 0x51, 0xcf, 0x67, 0x48, 0xce, 0x14, 0x16, 0xf5, 0x59, 0x21, 0xe8, 0x3a, 0xa7,
0xb6, 0xae, 0x75, 0xb1, 0x2b, 0xea, 0xed, 0x79, 0xff, 0xfa, 0xe5, 0xb7, 0xa1, 0xb0, 0x57, 0x12, 0x51, 0xab, 0xec, 0x0a, 0x66, 0xa8, 0x4a, 0x1b, 0x5b, 0xa2, 0xdf, 0x9e, 0x75, 0xae, 0x5f, 0xce,
0x5c, 0xa5, 0xaf, 0x56, 0x70, 0x95, 0x9f, 0x8f, 0xe0, 0xfa, 0x09, 0xa8, 0xb9, 0x81, 0xd4, 0xaa, 0x18, 0xf2, 0xa4, 0x21, 0xc2, 0x95, 0xfb, 0x6a, 0x09, 0x57, 0xfe, 0xd9, 0x10, 0xae, 0x1f, 0x81,
0x70, 0xca, 0x6b, 0x85, 0x36, 0xb6, 0x50, 0x59, 0x21, 0x75, 0xa8, 0xaf, 0x42, 0xba, 0x2c, 0x65, 0x92, 0xe5, 0x52, 0xad, 0x02, 0x87, 0xbc, 0x92, 0xa9, 0xb0, 0x05, 0xcb, 0xf2, 0xa0, 0x3d, 0x7e,
0x55, 0xfd, 0x3a, 0x95, 0x15, 0x4b, 0x74, 0x1b, 0x7b, 0x2e, 0xe9, 0xf1, 0x4d, 0x55, 0x8b, 0x12, 0xe5, 0xc1, 0x25, 0x31, 0xab, 0xe2, 0xd7, 0xc9, 0xac, 0x58, 0xa2, 0x1b, 0xd8, 0xb6, 0x48, 0x87,
0x7d, 0x97, 0xb7, 0x22, 0xd1, 0x0b, 0xf7, 0xc1, 0xa2, 0xed, 0x58, 0x7d, 0x87, 0xb8, 0xee, 0x06, 0x17, 0x55, 0xc9, 0x4f, 0xf4, 0x16, 0x1f, 0x45, 0x42, 0x0a, 0x0f, 0xc0, 0xb2, 0x61, 0xea, 0x5d,
0xc1, 0x3d, 0x5d, 0x33, 0x49, 0x10, 0x80, 0x5f, 0x13, 0x2f, 0x8f, 0x8e, 0x97, 0x16, 0x77, 0xb3, 0x93, 0x58, 0xd6, 0x16, 0xc1, 0x1d, 0x55, 0xd1, 0x88, 0x1b, 0x80, 0xd3, 0x13, 0x2f, 0x8c, 0x86,
0x21, 0x28, 0xcf, 0xb6, 0xf5, 0x97, 0x0a, 0xb8, 0x90, 0x3c, 0x1b, 0x73, 0x64, 0x8a, 0x72, 0x2a, 0xb5, 0xe5, 0x56, 0xb2, 0x0a, 0x4a, 0xb3, 0x95, 0xff, 0x5c, 0x00, 0xe7, 0xa2, 0x67, 0x63, 0x0a,
0x99, 0x72, 0x3d, 0x96, 0xa7, 0xbe, 0x86, 0x8b, 0x3d, 0x15, 0xa4, 0x72, 0x75, 0x0d, 0xcc, 0x0a, 0x4d, 0x91, 0x4e, 0x44, 0x53, 0xae, 0x06, 0xf2, 0xd4, 0xe1, 0x70, 0x81, 0xa7, 0x82, 0x58, 0xae,
0x59, 0x12, 0x74, 0x0a, 0xa1, 0x16, 0x2e, 0xcf, 0xbe, 0xdc, 0x8d, 0x92, 0x78, 0x78, 0x1b, 0x4c, 0x6e, 0x80, 0x79, 0x41, 0x4b, 0x5c, 0xa1, 0x20, 0x6a, 0xde, 0xf6, 0x1c, 0x84, 0xc5, 0x28, 0xaa,
0x3b, 0x5c, 0x79, 0x05, 0x04, 0xbe, 0x7a, 0xf9, 0x3f, 0x41, 0x30, 0x8d, 0xe2, 0x9d, 0x48, 0xc6, 0x0f, 0x6f, 0x82, 0x59, 0x93, 0x33, 0x2f, 0x17, 0xc0, 0x61, 0x2f, 0xff, 0x27, 0x00, 0x66, 0x51,
0x32, 0xe5, 0x12, 0x09, 0x92, 0x80, 0xa0, 0x22, 0x2b, 0x97, 0xb5, 0x24, 0x00, 0xa5, 0x6d, 0xe0, 0x50, 0x88, 0xc2, 0xba, 0x8c, 0xb9, 0xf8, 0x84, 0xc4, 0x05, 0x28, 0x84, 0x99, 0xcb, 0x46, 0x54,
0x16, 0x98, 0xf3, 0xcc, 0x34, 0x95, 0x9f, 0x6b, 0x97, 0x05, 0xd5, 0xdc, 0x7e, 0x1a, 0x82, 0xb2, 0x01, 0xc5, 0x6d, 0xe0, 0x0e, 0x58, 0xb0, 0xb5, 0x38, 0x94, 0x93, 0x6b, 0x17, 0x04, 0xd4, 0xc2,
0xec, 0xe0, 0x43, 0x49, 0xcc, 0x4c, 0xf0, 0xf3, 0xe4, 0x7a, 0x81, 0x3d, 0x51, 0x58, 0xcd, 0x64, 0x41, 0x5c, 0x05, 0x25, 0xd9, 0xc1, 0xfb, 0x21, 0x32, 0x33, 0xc5, 0xcf, 0x93, 0xab, 0x19, 0x6a,
0x48, 0xad, 0x5a, 0x51, 0xa9, 0xd5, 0xfa, 0x48, 0x01, 0x30, 0xbd, 0x0f, 0xc7, 0xbe, 0x04, 0xa4, 0x22, 0x33, 0x9b, 0x49, 0xa0, 0x5a, 0xa5, 0xac, 0x54, 0x4b, 0xfe, 0x48, 0x02, 0x30, 0x5e, 0x87,
0x2c, 0x62, 0x15, 0x53, 0xcb, 0xd6, 0x3f, 0x37, 0x0a, 0xea, 0x9f, 0xe8, 0x40, 0x2d, 0x26, 0x80, 0x63, 0x5f, 0x02, 0x62, 0x16, 0x81, 0x8e, 0xa9, 0x24, 0xf3, 0x9f, 0x6b, 0x19, 0xf9, 0x8f, 0x7f,
0xc4, 0x44, 0x9f, 0xcd, 0xa3, 0x4e, 0x51, 0x01, 0x14, 0x39, 0xf5, 0x1c, 0x04, 0x50, 0x8c, 0xec, 0xa0, 0x66, 0x23, 0x40, 0x62, 0xa1, 0x4f, 0xe7, 0x51, 0x27, 0x2b, 0x01, 0xf2, 0x9d, 0x7a, 0x06,
0xd9, 0x02, 0xe8, 0x9f, 0x25, 0x30, 0x17, 0x81, 0x0b, 0x0b, 0xa0, 0x0c, 0x93, 0xef, 0x1e, 0x76, 0x04, 0x28, 0x00, 0xf6, 0x74, 0x02, 0xf4, 0x8f, 0x1c, 0x58, 0xf0, 0x95, 0x33, 0x13, 0xa0, 0x04,
0x8a, 0x89, 0x92, 0x68, 0xea, 0xfe, 0x97, 0x44, 0x49, 0xe4, 0x55, 0x8e, 0x28, 0xf9, 0x43, 0x29, 0x93, 0xef, 0x1e, 0x76, 0xb2, 0x91, 0x12, 0x7f, 0xe9, 0xfe, 0x97, 0x48, 0x89, 0xef, 0x55, 0x0a,
0xee, 0xfa, 0x09, 0x45, 0xc9, 0x73, 0x78, 0xe1, 0xf8, 0xc6, 0xe9, 0x9a, 0xd6, 0x5f, 0xcb, 0xe0, 0x29, 0xf9, 0x7d, 0x2e, 0xe8, 0xfa, 0x84, 0xa4, 0xe4, 0x19, 0xbc, 0x70, 0x7c, 0xe3, 0x78, 0x8d,
0x42, 0x72, 0x1f, 0x4a, 0x05, 0x52, 0x19, 0x5b, 0x20, 0x77, 0xc1, 0xfc, 0x23, 0x4f, 0xd7, 0x87, 0xfc, 0x49, 0x1e, 0x9c, 0x8b, 0xd6, 0x61, 0xa8, 0x41, 0x4a, 0x63, 0x1b, 0x64, 0x0b, 0x2c, 0x3e,
0x3c, 0x86, 0x58, 0x95, 0xf4, 0x4b, 0xeb, 0xff, 0x0b, 0xcb, 0xf9, 0x1f, 0x67, 0x60, 0x50, 0xa6, 0xb0, 0x55, 0x75, 0xc0, 0x63, 0x08, 0x74, 0x49, 0xa7, 0xb5, 0xfe, 0xbf, 0xb0, 0x5c, 0xfc, 0x61,
0x65, 0xba, 0x5e, 0x56, 0xbe, 0x6c, 0xbd, 0xac, 0x9e, 0xa2, 0x5e, 0x66, 0x4b, 0x8e, 0xf2, 0xa9, 0x82, 0x0e, 0x4a, 0xb4, 0x8c, 0xf7, 0xcb, 0xc2, 0x97, 0xed, 0x97, 0xc5, 0x13, 0xf4, 0xcb, 0x64,
0x24, 0xc7, 0xc9, 0x8a, 0x65, 0xc6, 0xc1, 0x35, 0xf6, 0xea, 0x3f, 0x52, 0xc0, 0x42, 0xf6, 0x85, 0xca, 0x91, 0x3f, 0x11, 0xe5, 0x98, 0xac, 0x59, 0x26, 0x1c, 0x5c, 0x63, 0xaf, 0xfe, 0x23, 0x09,
0x1b, 0xea, 0x60, 0xc6, 0xc0, 0x8f, 0xe3, 0x0f, 0x1f, 0xe3, 0x8a, 0x88, 0x47, 0x35, 0x5d, 0xf5, 0x2c, 0x25, 0x5f, 0xb8, 0xa1, 0x0a, 0xe6, 0x7a, 0xf8, 0x61, 0xf0, 0xe1, 0x63, 0x5c, 0x13, 0xb1,
0x3f, 0x19, 0xa9, 0xf7, 0x4c, 0xba, 0xe3, 0xec, 0x51, 0x47, 0x33, 0xfb, 0x7e, 0xe5, 0xdd, 0x92, 0xa9, 0xa2, 0xd6, 0x9d, 0x4f, 0x46, 0xf5, 0x3b, 0x1a, 0xdd, 0x33, 0xf7, 0xa9, 0xa9, 0x68, 0x5d,
0xb8, 0x50, 0x82, 0x1b, 0xbe, 0x0f, 0x6a, 0x06, 0x7e, 0xbc, 0xe7, 0x39, 0xfd, 0xac, 0x0a, 0x59, 0xa7, 0xf3, 0xee, 0x84, 0xb0, 0x50, 0x04, 0x1b, 0xbe, 0x0f, 0x4a, 0x3d, 0xfc, 0x70, 0xdf, 0x36,
0x6c, 0x1c, 0xbe, 0x01, 0xb6, 0x04, 0x0b, 0x0a, 0xf9, 0x5a, 0x9f, 0x2b, 0x60, 0x31, 0xa7, 0xaa, 0xbb, 0x49, 0x1d, 0x32, 0xdb, 0x3c, 0xbc, 0x00, 0x76, 0x04, 0x0a, 0xf2, 0xf0, 0xe4, 0x2f, 0x24,
0x7e, 0x8b, 0xa2, 0xdc, 0x01, 0x57, 0xa4, 0x20, 0xd9, 0xae, 0x24, 0x8f, 0x3c, 0x9d, 0x6f, 0x50, 0xb0, 0x9c, 0xd2, 0x55, 0xbf, 0x45, 0x51, 0xee, 0x81, 0x8b, 0xa1, 0x20, 0x59, 0x55, 0x92, 0x07,
0x21, 0x64, 0xae, 0x81, 0xba, 0x8d, 0x1d, 0xaa, 0x85, 0x32, 0xb8, 0xda, 0x99, 0x1e, 0x1d, 0x2f, 0xb6, 0xca, 0x0b, 0x54, 0x10, 0x99, 0x2b, 0xa0, 0x6c, 0x60, 0x93, 0x2a, 0x1e, 0x0d, 0x2e, 0x36,
0xd5, 0x77, 0x83, 0x46, 0x14, 0xf5, 0xb7, 0xfe, 0xa3, 0x80, 0xea, 0x5e, 0x17, 0xeb, 0xe4, 0x0c, 0x67, 0x47, 0xc3, 0x5a, 0xb9, 0xe5, 0x0e, 0x22, 0x5f, 0x2e, 0xff, 0x47, 0x02, 0xc5, 0xfd, 0x36,
0x94, 0xc4, 0x86, 0xa4, 0x24, 0xf2, 0x5f, 0xe9, 0xb9, 0x3f, 0xb9, 0x22, 0x62, 0x33, 0x21, 0x22, 0x56, 0xc9, 0x29, 0x30, 0x89, 0xad, 0x10, 0x93, 0x48, 0x7f, 0xa5, 0xe7, 0xfe, 0xa4, 0x92, 0x88,
0x5e, 0x1a, 0xc3, 0xf3, 0x6c, 0xfd, 0xf0, 0x36, 0xa8, 0x87, 0xc3, 0x9d, 0xec, 0x70, 0x6b, 0xfd, 0xed, 0x08, 0x89, 0xb8, 0x34, 0x06, 0xe7, 0xe9, 0xfc, 0xe1, 0x0d, 0x50, 0xf6, 0xa6, 0x9b, 0xec,
0xbe, 0x04, 0xa6, 0x62, 0x43, 0x9c, 0xf0, 0x68, 0x7c, 0x28, 0xd5, 0x03, 0xb6, 0xe9, 0x57, 0x8b, 0x70, 0x93, 0x7f, 0x97, 0x03, 0x33, 0x81, 0x29, 0x26, 0x3c, 0x1a, 0xef, 0x87, 0xfa, 0x01, 0x2b,
0x04, 0xa2, 0x06, 0x67, 0xff, 0x3b, 0x26, 0x75, 0xe2, 0x97, 0xc7, 0x74, 0x49, 0xf8, 0x01, 0x98, 0xfa, 0xf5, 0x2c, 0x81, 0xd4, 0xdd, 0xb3, 0xff, 0x6d, 0x8d, 0x9a, 0xc1, 0xcb, 0x63, 0xbc, 0x25,
0xa1, 0xd8, 0xe9, 0x13, 0x1a, 0xf4, 0xf1, 0x09, 0xab, 0x47, 0x8f, 0x29, 0xf7, 0xa5, 0x5e, 0x94, 0xbc, 0x09, 0xe6, 0x28, 0x36, 0xbb, 0x84, 0xba, 0x32, 0xbe, 0x60, 0x65, 0xff, 0x31, 0xe5, 0x6e,
0x40, 0x5f, 0xba, 0x0d, 0xa6, 0xa5, 0xc1, 0xe0, 0x05, 0x50, 0x3e, 0x22, 0x43, 0x5f, 0x52, 0x21, 0x48, 0x8a, 0x22, 0xda, 0x2b, 0x37, 0xc1, 0x6c, 0x68, 0x32, 0x78, 0x0e, 0xe4, 0x8f, 0xc9, 0xc0,
0xf6, 0x27, 0x9c, 0x07, 0xd5, 0x01, 0xd6, 0x3d, 0x3f, 0xcf, 0xeb, 0xc8, 0xff, 0x71, 0xab, 0xf4, 0xa1, 0x54, 0x88, 0xfd, 0x09, 0x17, 0x41, 0xb1, 0x8f, 0x55, 0xdb, 0xc9, 0xf3, 0x32, 0x72, 0x7e,
0x96, 0xd2, 0xfa, 0x2d, 0x9b, 0x9c, 0x28, 0x39, 0xcf, 0x20, 0xbb, 0xde, 0x95, 0xb2, 0x2b, 0xff, 0xdc, 0xc8, 0xbd, 0x2e, 0xc9, 0xbf, 0x66, 0x8b, 0xe3, 0x27, 0xe7, 0x29, 0x64, 0xd7, 0x3b, 0xa1,
0x83, 0x61, 0x7c, 0xcb, 0xe4, 0xe5, 0x18, 0x4a, 0xe4, 0xd8, 0xab, 0x85, 0xd8, 0x9e, 0x9d, 0x69, 0xec, 0x4a, 0xff, 0x60, 0x18, 0x2c, 0x99, 0xb4, 0x1c, 0x43, 0x91, 0x1c, 0x7b, 0x29, 0x13, 0xda,
0xff, 0x2a, 0x81, 0xf9, 0x18, 0x3a, 0x92, 0xaa, 0xdf, 0x97, 0xa4, 0xea, 0x72, 0x42, 0xaa, 0x36, 0xd3, 0x33, 0xed, 0x9f, 0x39, 0xb0, 0x18, 0xd0, 0xf6, 0xa9, 0xea, 0xf7, 0x43, 0x54, 0x75, 0x35,
0xb2, 0x6c, 0xbe, 0xd3, 0xaa, 0xe3, 0xb5, 0xea, 0x9f, 0x15, 0x30, 0x1b, 0x9b, 0xbb, 0x33, 0x10, 0x42, 0x55, 0x2b, 0x49, 0x36, 0xdf, 0x71, 0xd5, 0xf1, 0x5c, 0xf5, 0x4f, 0x12, 0x98, 0x0f, 0xac,
0xab, 0xf7, 0x64, 0xb1, 0xfa, 0x52, 0x91, 0xa4, 0xc9, 0x51, 0xab, 0x1f, 0x4e, 0x48, 0xce, 0x7f, 0xdd, 0x29, 0x90, 0xd5, 0x3b, 0x61, 0xb2, 0x7a, 0x29, 0x4b, 0xd2, 0xa4, 0xb0, 0xd5, 0x7f, 0x49,
0xeb, 0xdf, 0xd0, 0x7e, 0x09, 0xe6, 0x07, 0x96, 0xee, 0x19, 0x64, 0x5d, 0xc7, 0x9a, 0x11, 0x00, 0xa0, 0x11, 0xd0, 0x6a, 0x11, 0xd3, 0x52, 0x2c, 0x4a, 0x34, 0x7a, 0x4f, 0x57, 0xed, 0x1e, 0xd9,
0x98, 0xba, 0x2b, 0x27, 0xef, 0x89, 0x21, 0x3d, 0x71, 0x5c, 0xcd, 0xa5, 0xc4, 0xa4, 0x0f, 0x22, 0x54, 0xb1, 0xd2, 0x43, 0x84, 0x0d, 0x28, 0xba, 0xd6, 0xd2, 0x55, 0xa5, 0x3d, 0x80, 0x18, 0xcc,
0xcb, 0x48, 0x53, 0x3e, 0xc8, 0xa0, 0x43, 0x99, 0x83, 0xc0, 0x37, 0xc0, 0x14, 0x53, 0x65, 0x5a, 0x7c, 0x78, 0x44, 0xb4, 0x2d, 0xa2, 0x12, 0x2a, 0x3e, 0x6b, 0x95, 0x9b, 0x6f, 0xb9, 0x5f, 0x79,
0x97, 0x6c, 0x63, 0x23, 0x48, 0xac, 0xf0, 0xf3, 0xd8, 0x5e, 0xd4, 0x85, 0xe2, 0x38, 0x78, 0x08, 0xde, 0xf3, 0x45, 0x4f, 0x86, 0xb5, 0xd5, 0x2c, 0x88, 0x3c, 0xcb, 0x82, 0x98, 0xf0, 0xa7, 0x00,
0xe6, 0x6c, 0xab, 0xb7, 0x85, 0x4d, 0xdc, 0x27, 0x4c, 0x66, 0xec, 0x5a, 0xba, 0xd6, 0x1d, 0xf2, 0xb0, 0x9f, 0xfc, 0x3c, 0xea, 0x88, 0x84, 0x7b, 0xd3, 0xad, 0xca, 0xf7, 0x3c, 0xc9, 0x44, 0x13,
0x87, 0xb5, 0x7a, 0xe7, 0xcd, 0xe0, 0xd1, 0x64, 0x37, 0x0d, 0x61, 0x17, 0xd0, 0x8c, 0x66, 0xbe, 0x04, 0x10, 0xe5, 0xbf, 0x4e, 0x87, 0xf6, 0xec, 0x5b, 0xff, 0x74, 0xf8, 0x73, 0xb0, 0xd8, 0xf7,
0xa9, 0xb3, 0x28, 0xa1, 0x93, 0xfa, 0xa4, 0xeb, 0x3f, 0x69, 0xaf, 0x16, 0xc9, 0xb0, 0x53, 0x7e, 0x57, 0xc7, 0x55, 0x60, 0xa4, 0x36, 0x1f, 0xbd, 0x1e, 0x7b, 0xf0, 0x49, 0xeb, 0xea, 0x53, 0xe9,
0xd4, 0xcd, 0x7b, 0x37, 0xac, 0x9d, 0xea, 0xdd, 0x30, 0xe3, 0x02, 0x55, 0x3f, 0xd9, 0x05, 0xaa, 0x7b, 0x09, 0x70, 0x28, 0x71, 0x12, 0xf8, 0x2a, 0x98, 0x61, 0x64, 0x54, 0x69, 0x93, 0x5d, 0xdc,
0xf5, 0x51, 0x15, 0x5c, 0x4c, 0x9d, 0xb6, 0x5f, 0xe3, 0xe3, 0x5f, 0xea, 0x26, 0x52, 0x3e, 0xc1, 0x73, 0xeb, 0xc9, 0xfb, 0x2a, 0xb8, 0xef, 0x8b, 0x50, 0x50, 0x0f, 0x1e, 0x81, 0x05, 0x43, 0xef,
0x4d, 0x64, 0x0d, 0xcc, 0x8a, 0xef, 0xc9, 0x89, 0x8b, 0x4c, 0x38, 0x1f, 0xeb, 0x72, 0x37, 0x4a, 0xec, 0x60, 0x0d, 0x77, 0x09, 0x63, 0x57, 0xce, 0x56, 0xf2, 0xf7, 0xc4, 0x72, 0xf3, 0x35, 0xf7,
0xe2, 0xb3, 0x1e, 0x1f, 0xab, 0x27, 0x7c, 0x7c, 0x8c, 0x7b, 0x21, 0xfe, 0x3f, 0xca, 0xcf, 0xde, 0xad, 0xa8, 0x15, 0x57, 0x61, 0xf7, 0xee, 0x84, 0x61, 0x9e, 0x04, 0x49, 0x90, 0xd0, 0x8c, 0x7d,
0xb4, 0x17, 0xe2, 0xdf, 0xa4, 0x92, 0x78, 0x26, 0x32, 0x7c, 0xd6, 0x90, 0x61, 0x52, 0x16, 0x19, 0xc9, 0x76, 0x5e, 0xf2, 0xd7, 0xb3, 0x14, 0xd6, 0x09, 0xbf, 0x65, 0xa7, 0x3d, 0x97, 0x96, 0x4e,
0xfb, 0x52, 0x2f, 0x4a, 0xa0, 0xbf, 0xd4, 0x37, 0x53, 0x9c, 0xf1, 0xcd, 0x74, 0xa5, 0xc8, 0x96, 0xf4, 0x5c, 0x9a, 0x70, 0x6f, 0x2c, 0x4f, 0x78, 0x6f, 0xfc, 0x44, 0x02, 0x97, 0x8c, 0x0c, 0xb5,
0x28, 0xfe, 0xce, 0x98, 0x79, 0x63, 0x9c, 0x3a, 0xf9, 0x8d, 0xb1, 0xf5, 0x37, 0x05, 0xbc, 0x90, 0x54, 0x01, 0x7c, 0x6d, 0x6e, 0x67, 0x59, 0x9b, 0x2c, 0xb5, 0xd9, 0x5c, 0x1d, 0x0d, 0x6b, 0x97,
0xbb, 0x29, 0xe1, 0x9a, 0x24, 0x01, 0x56, 0x12, 0x12, 0xe0, 0x7b, 0xb9, 0x86, 0x31, 0x1d, 0xe0, 0xb2, 0x68, 0xa2, 0x4c, 0xfe, 0xc9, 0x1f, 0x15, 0xc1, 0xf9, 0x58, 0xb7, 0xfc, 0x1a, 0x1f, 0x6f,
0x64, 0x3f, 0x41, 0xbe, 0x5d, 0xec, 0x09, 0x32, 0xe3, 0x1e, 0x31, 0xfe, 0x2d, 0xb2, 0xb3, 0xf2, 0x63, 0x37, 0xc9, 0xfc, 0x04, 0x37, 0xc9, 0x0d, 0x30, 0x2f, 0xfe, 0x1f, 0x20, 0x72, 0x11, 0xf5,
0xe4, 0x69, 0xf3, 0xdc, 0x27, 0x4f, 0x9b, 0xe7, 0x3e, 0x7b, 0xda, 0x3c, 0xf7, 0xab, 0x51, 0x53, 0x36, 0x76, 0x33, 0x2c, 0x46, 0x51, 0xfd, 0xa4, 0xc7, 0xe3, 0xe2, 0x84, 0x8f, 0xc7, 0x41, 0x2f,
0x79, 0x32, 0x6a, 0x2a, 0x9f, 0x8c, 0x9a, 0xca, 0x67, 0xa3, 0xa6, 0xf2, 0xf7, 0x51, 0x53, 0xf9, 0xc4, 0xff, 0xb7, 0x39, 0x65, 0x18, 0xf7, 0x42, 0xfc, 0x9b, 0x5b, 0x54, 0x9f, 0x91, 0x44, 0x07,
0xf0, 0xf3, 0xe6, 0xb9, 0xf7, 0x27, 0xc5, 0x88, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xda, 0x5d, 0xd5, 0x43, 0x98, 0x0e, 0x93, 0xc4, 0x83, 0x90, 0x14, 0x45, 0xb4, 0xbf, 0xd4, 0x37, 0x6f, 0x9c,
0xee, 0xc9, 0xd4, 0x29, 0x00, 0x00, 0xf0, 0xcd, 0x7b, 0x2d, 0x4b, 0xfe, 0x66, 0x7f, 0x27, 0x4e, 0xbc, 0xf1, 0xcf, 0x4c, 0x7e, 0xe3,
0x97, 0xff, 0x22, 0x81, 0xe7, 0x52, 0x4f, 0x17, 0xb8, 0x11, 0xa2, 0x70, 0x6b, 0x11, 0x0a, 0xf7,
0xbd, 0x54, 0xc3, 0x00, 0x8f, 0x33, 0x93, 0x9f, 0x90, 0xdf, 0xc8, 0xf6, 0x84, 0x9c, 0x70, 0x0f,
0x1c, 0xff, 0x96, 0xdc, 0x5c, 0x7b, 0xf4, 0xb8, 0x7a, 0xe6, 0xd3, 0xc7, 0xd5, 0x33, 0x9f, 0x3f,
0xae, 0x9e, 0xf9, 0xc5, 0xa8, 0x2a, 0x3d, 0x1a, 0x55, 0xa5, 0x4f, 0x47, 0x55, 0xe9, 0xf3, 0x51,
0x55, 0xfa, 0xdb, 0xa8, 0x2a, 0xfd, 0xe6, 0x8b, 0xea, 0x99, 0xf7, 0xa7, 0xc5, 0x8c, 0xff, 0x0d,
0x00, 0x00, 0xff, 0xff, 0x3e, 0x13, 0x3b, 0xc7, 0x94, 0x2b, 0x00, 0x00,
} }
func (m *ControllerRevision) Marshal() (dAtA []byte, err error) { func (m *ControllerRevision) Marshal() (dAtA []byte, err error) {
@ -2523,6 +2560,39 @@ func (m *StatefulSetList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
i -= len(m.WhenScaled)
copy(dAtA[i:], m.WhenScaled)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenScaled)))
i--
dAtA[i] = 0x12
i -= len(m.WhenDeleted)
copy(dAtA[i:], m.WhenDeleted)
i = encodeVarintGenerated(dAtA, i, uint64(len(m.WhenDeleted)))
i--
dAtA[i] = 0xa
return len(dAtA) - i, nil
}
func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) { func (m *StatefulSetSpec) Marshal() (dAtA []byte, err error) {
size := m.Size() size := m.Size()
dAtA = make([]byte, size) dAtA = make([]byte, size)
@ -2543,6 +2613,18 @@ func (m *StatefulSetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if m.PersistentVolumeClaimRetentionPolicy != nil {
{
size, err := m.PersistentVolumeClaimRetentionPolicy.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintGenerated(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x52
}
i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds))
i-- i--
dAtA[i] = 0x48 dAtA[i] = 0x48
@ -3232,6 +3314,19 @@ func (m *StatefulSetList) Size() (n int) {
return n return n
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.WhenDeleted)
n += 1 + l + sovGenerated(uint64(l))
l = len(m.WhenScaled)
n += 1 + l + sovGenerated(uint64(l))
return n
}
func (m *StatefulSetSpec) Size() (n int) { func (m *StatefulSetSpec) Size() (n int) {
if m == nil { if m == nil {
return 0 return 0
@ -3263,6 +3358,10 @@ func (m *StatefulSetSpec) Size() (n int) {
n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit)) n += 1 + sovGenerated(uint64(*m.RevisionHistoryLimit))
} }
n += 1 + sovGenerated(uint64(m.MinReadySeconds)) n += 1 + sovGenerated(uint64(m.MinReadySeconds))
if m.PersistentVolumeClaimRetentionPolicy != nil {
l = m.PersistentVolumeClaimRetentionPolicy.Size()
n += 1 + l + sovGenerated(uint64(l))
}
return n return n
} }
@ -3720,6 +3819,17 @@ func (this *StatefulSetList) String() string {
}, "") }, "")
return s return s
} }
func (this *StatefulSetPersistentVolumeClaimRetentionPolicy) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&StatefulSetPersistentVolumeClaimRetentionPolicy{`,
`WhenDeleted:` + fmt.Sprintf("%v", this.WhenDeleted) + `,`,
`WhenScaled:` + fmt.Sprintf("%v", this.WhenScaled) + `,`,
`}`,
}, "")
return s
}
func (this *StatefulSetSpec) String() string { func (this *StatefulSetSpec) String() string {
if this == nil { if this == nil {
return "nil" return "nil"
@ -3739,6 +3849,7 @@ func (this *StatefulSetSpec) String() string {
`UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`, `UpdateStrategy:` + strings.Replace(strings.Replace(this.UpdateStrategy.String(), "StatefulSetUpdateStrategy", "StatefulSetUpdateStrategy", 1), `&`, ``, 1) + `,`,
`RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`, `RevisionHistoryLimit:` + valueToStringGenerated(this.RevisionHistoryLimit) + `,`,
`MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`, `MinReadySeconds:` + fmt.Sprintf("%v", this.MinReadySeconds) + `,`,
`PersistentVolumeClaimRetentionPolicy:` + strings.Replace(this.PersistentVolumeClaimRetentionPolicy.String(), "StatefulSetPersistentVolumeClaimRetentionPolicy", "StatefulSetPersistentVolumeClaimRetentionPolicy", 1) + `,`,
`}`, `}`,
}, "") }, "")
return s return s
@ -8228,6 +8339,120 @@ func (m *StatefulSetList) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *StatefulSetPersistentVolumeClaimRetentionPolicy) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: StatefulSetPersistentVolumeClaimRetentionPolicy: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenDeleted", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenDeleted = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field WhenScaled", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.WhenScaled = PersistentVolumeClaimRetentionPolicyType(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthGenerated
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error { func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0
@ -8516,6 +8741,42 @@ func (m *StatefulSetSpec) Unmarshal(dAtA []byte) error {
break break
} }
} }
case 10:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeClaimRetentionPolicy", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthGenerated
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.PersistentVolumeClaimRetentionPolicy == nil {
m.PersistentVolumeClaimRetentionPolicy = &StatefulSetPersistentVolumeClaimRetentionPolicy{}
}
if err := m.PersistentVolumeClaimRetentionPolicy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:]) skippy, err := skipGenerated(dAtA[iNdEx:])

View File

@ -656,6 +656,23 @@ message StatefulSetList {
repeated StatefulSet items = 2; repeated StatefulSet items = 2;
} }
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
message StatefulSetPersistentVolumeClaimRetentionPolicy {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
optional string whenDeleted = 1;
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
optional string whenScaled = 2;
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
message StatefulSetSpec { message StatefulSetSpec {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -722,6 +739,12 @@ message StatefulSetSpec {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
optional int32 minReadySeconds = 9; optional int32 minReadySeconds = 9;
// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
optional StatefulSetPersistentVolumeClaimRetentionPolicy persistentVolumeClaimRetentionPolicy = 10;
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -169,6 +169,40 @@ type RollingUpdateStatefulSetStrategy struct {
Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"` Partition *int32 `json:"partition,omitempty" protobuf:"varint,1,opt,name=partition"`
} }
// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
// deleted or scaled down.
type PersistentVolumeClaimRetentionPolicyType string
const (
// RetainPersistentVolumeClaimRetentionPolicyType is the default
// PersistentVolumeClaimRetentionPolicy and specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will not be deleted.
RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
// RetentionPersistentVolumeClaimRetentionPolicyType specifies that
// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
// will be deleted in the scenario specified in
// StatefulSetPersistentVolumeClaimRetentionPolicy.
RetentionPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
)
// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
// created from the StatefulSet VolumeClaimTemplates.
type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
// WhenDeleted specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
// `Delete` policy causes those PVCs to be deleted.
WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty" protobuf:"bytes,1,opt,name=whenDeleted,casttype=PersistentVolumeClaimRetentionPolicyType"`
// WhenScaled specifies what happens to PVCs created from StatefulSet
// VolumeClaimTemplates when the StatefulSet is scaled down. The default
// policy of `Retain` causes PVCs to not be affected by a scaledown. The
// `Delete` policy causes the associated PVCs for any excess pods above
// the replica count to be deleted.
WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty" protobuf:"bytes,2,opt,name=whenScaled,casttype=PersistentVolumeClaimRetentionPolicyType"`
}
// A StatefulSetSpec is the specification of a StatefulSet. // A StatefulSetSpec is the specification of a StatefulSet.
type StatefulSetSpec struct { type StatefulSetSpec struct {
// replicas is the desired number of replicas of the given Template. // replicas is the desired number of replicas of the given Template.
@ -235,6 +269,12 @@ type StatefulSetSpec struct {
// This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate. // This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.
// +optional // +optional
MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"` MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,9,opt,name=minReadySeconds"`
// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
// the StatefulSet VolumeClaimTemplates. This requires the
// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
// +optional
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaimRetentionPolicy"`
} }
// StatefulSetStatus represents the current state of a StatefulSet. // StatefulSetStatus represents the current state of a StatefulSet.

View File

@ -351,17 +351,28 @@ func (StatefulSetList) SwaggerDoc() map[string]string {
return map_StatefulSetList return map_StatefulSetList
} }
var map_StatefulSetPersistentVolumeClaimRetentionPolicy = map[string]string{
"": "StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates.",
"whenDeleted": "WhenDeleted specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is deleted. The default policy of `Retain` causes PVCs to not be affected by StatefulSet deletion. The `Delete` policy causes those PVCs to be deleted.",
"whenScaled": "WhenScaled specifies what happens to PVCs created from StatefulSet VolumeClaimTemplates when the StatefulSet is scaled down. The default policy of `Retain` causes PVCs to not be affected by a scaledown. The `Delete` policy causes the associated PVCs for any excess pods above the replica count to be deleted.",
}
func (StatefulSetPersistentVolumeClaimRetentionPolicy) SwaggerDoc() map[string]string {
return map_StatefulSetPersistentVolumeClaimRetentionPolicy
}
var map_StatefulSetSpec = map[string]string{ var map_StatefulSetSpec = map[string]string{
"": "A StatefulSetSpec is the specification of a StatefulSet.", "": "A StatefulSetSpec is the specification of a StatefulSet.",
"replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.", "replicas": "replicas is the desired number of replicas of the given Template. These are replicas in the sense that they are instantiations of the same Template, but individual replicas also have a consistent identity. If unspecified, defaults to 1.",
"selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors", "selector": "selector is a label query over pods that should match the replica count. It must match the pod template's labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors",
"template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.", "template": "template is the object that describes the pod that will be created if insufficient replicas are detected. Each pod stamped out by the StatefulSet will fulfill this Template, but have a unique identity from the rest of the StatefulSet.",
"volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.", "volumeClaimTemplates": "volumeClaimTemplates is a list of claims that pods are allowed to reference. The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. A claim in this list takes precedence over any volumes in the template, with the same name.",
"serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.", "serviceName": "serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where \"pod-specific-string\" is managed by the StatefulSet controller.",
"podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.", "podManagementPolicy": "podManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down. The default policy is `OrderedReady`, where pods are created in increasing order (pod-0, then pod-1, etc) and the controller will wait until each pod is ready before continuing. When scaling down, the pods are removed in the opposite order. The alternative policy is `Parallel` which will create pods in parallel to match the desired scale without waiting, and on scale down will delete all pods at once.",
"updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.", "updateStrategy": "updateStrategy indicates the StatefulSetUpdateStrategy that will be employed to update Pods in the StatefulSet when a revision is made to Template.",
"revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.", "revisionHistoryLimit": "revisionHistoryLimit is the maximum number of revisions that will be maintained in the StatefulSet's revision history. The revision history consists of all revisions not represented by a currently applied StatefulSetSpec version. The default value is 10.",
"minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.", "minReadySeconds": "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready) This is an alpha field and requires enabling StatefulSetMinReadySeconds feature gate.",
"persistentVolumeClaimRetentionPolicy": "PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from the StatefulSet VolumeClaimTemplates. This requires the StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.",
} }
func (StatefulSetSpec) SwaggerDoc() map[string]string { func (StatefulSetSpec) SwaggerDoc() map[string]string {

View File

@ -755,6 +755,22 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object {
return nil return nil
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy.
func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy {
if in == nil {
return nil
}
out := new(StatefulSetPersistentVolumeClaimRetentionPolicy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = *in *out = *in
@ -782,6 +798,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = new(int32) *out = new(int32)
**out = **in **out = **in
} }
if in.PersistentVolumeClaimRetentionPolicy != nil {
in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy
*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
**out = **in
}
return return
} }

View File

@ -1693,26 +1693,30 @@
} }
}, },
"revisionHistoryLimit": -1166275743, "revisionHistoryLimit": -1166275743,
"minReadySeconds": -1056262432 "minReadySeconds": -1056262432,
"persistentVolumeClaimRetentionPolicy": {
"whenDeleted": "/Ş槀墺=Ĉ鳟/d",
"whenScaled": "á"
}
}, },
"status": { "status": {
"observedGeneration": -3770279213092072518, "observedGeneration": 5897193066236579633,
"replicas": -1669166259, "replicas": 1442750499,
"readyReplicas": -175399547, "readyReplicas": -1190434752,
"currentReplicas": 223338274, "currentReplicas": -1929177074,
"updatedReplicas": -1279136912, "updatedReplicas": -690843774,
"currentRevision": "538", "currentRevision": "538",
"updateRevision": "539", "updateRevision": "539",
"collisionCount": 1222237461, "collisionCount": 1329525670,
"conditions": [ "conditions": [
{ {
"type": "壣V礆á¤拈tY", "type": "\u0026嬜Š\u0026?鳢.ǀŭ瘢",
"status": "飼蒱鄆\u0026嬜Š\u0026?鳢.ǀŭ瘢颦z", "status": "Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6",
"lastTransitionTime": "2368-07-30T22:05:09Z", "lastTransitionTime": "2881-11-22T04:01:01Z",
"reason": "540", "reason": "540",
"message": "541" "message": "541"
} }
], ],
"availableReplicas": -1174483980 "availableReplicas": -1889018254
} }
} }

View File

@ -32,6 +32,9 @@ metadata:
uid: "7" uid: "7"
spec: spec:
minReadySeconds: -1056262432 minReadySeconds: -1056262432
persistentVolumeClaimRetentionPolicy:
whenDeleted: /Ş槀墺=Ĉ鳟/d
whenScaled: á
podManagementPolicy: Ȍ%ÿ¼璤ňɈ podManagementPolicy: Ȍ%ÿ¼璤ňɈ
replicas: 896585016 replicas: 896585016
revisionHistoryLimit: -1166275743 revisionHistoryLimit: -1166275743
@ -1164,18 +1167,18 @@ spec:
phase: ñƍU烈 źfjǰɪ phase: ñƍU烈 źfjǰɪ
resizeStatus: ʨɺC`牯 resizeStatus: ʨɺC`牯
status: status:
availableReplicas: -1174483980 availableReplicas: -1889018254
collisionCount: 1222237461 collisionCount: 1329525670
conditions: conditions:
- lastTransitionTime: "2368-07-30T22:05:09Z" - lastTransitionTime: "2881-11-22T04:01:01Z"
message: "541" message: "541"
reason: "540" reason: "540"
status: 飼蒱鄆&嬜Š&?鳢.ǀŭ瘢颦z status: Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6
type: 壣V礆á¤拈tY type: '&嬜Š&?鳢.ǀŭ瘢'
currentReplicas: 223338274 currentReplicas: -1929177074
currentRevision: "538" currentRevision: "538"
observedGeneration: -3770279213092072518 observedGeneration: 5897193066236579633
readyReplicas: -175399547 readyReplicas: -1190434752
replicas: -1669166259 replicas: 1442750499
updateRevision: "539" updateRevision: "539"
updatedReplicas: -1279136912 updatedReplicas: -690843774

View File

@ -1693,26 +1693,30 @@
} }
}, },
"revisionHistoryLimit": -1166275743, "revisionHistoryLimit": -1166275743,
"minReadySeconds": -1056262432 "minReadySeconds": -1056262432,
"persistentVolumeClaimRetentionPolicy": {
"whenDeleted": "/Ş槀墺=Ĉ鳟/d",
"whenScaled": "á"
}
}, },
"status": { "status": {
"observedGeneration": 1124654959171263717, "observedGeneration": 6696165662757041544,
"replicas": -1775998279, "replicas": -629510776,
"readyReplicas": 233229473, "readyReplicas": -69450448,
"currentReplicas": 99917513, "currentReplicas": -212409426,
"updatedReplicas": 421164481, "updatedReplicas": 17761427,
"currentRevision": "538", "currentRevision": "538",
"updateRevision": "539", "updateRevision": "539",
"collisionCount": -1137929768, "collisionCount": -168103644,
"conditions": [ "conditions": [
{ {
"type": "/d\u0026蒡榤Ⱦ盜ŭ飼蒱鄆", "type": "",
"status": "", "status": "嬜Š\u0026?鳢.ǀŭ瘢颦z疵",
"lastTransitionTime": "2358-03-12T13:17:54Z", "lastTransitionTime": "2455-07-16T22:37:15Z",
"reason": "540", "reason": "540",
"message": "541" "message": "541"
} }
], ],
"availableReplicas": 171558604 "availableReplicas": 2136031303
} }
} }

View File

@ -32,6 +32,9 @@ metadata:
uid: "7" uid: "7"
spec: spec:
minReadySeconds: -1056262432 minReadySeconds: -1056262432
persistentVolumeClaimRetentionPolicy:
whenDeleted: /Ş槀墺=Ĉ鳟/d
whenScaled: á
podManagementPolicy: Ȍ%ÿ¼璤ňɈ podManagementPolicy: Ȍ%ÿ¼璤ňɈ
replicas: 896585016 replicas: 896585016
revisionHistoryLimit: -1166275743 revisionHistoryLimit: -1166275743
@ -1164,18 +1167,18 @@ spec:
phase: ñƍU烈 źfjǰɪ phase: ñƍU烈 źfjǰɪ
resizeStatus: ʨɺC`牯 resizeStatus: ʨɺC`牯
status: status:
availableReplicas: 171558604 availableReplicas: 2136031303
collisionCount: -1137929768 collisionCount: -168103644
conditions: conditions:
- lastTransitionTime: "2358-03-12T13:17:54Z" - lastTransitionTime: "2455-07-16T22:37:15Z"
message: "541" message: "541"
reason: "540" reason: "540"
status: "" status: 嬜Š&?鳢.ǀŭ瘢颦z疵
type: /d&蒡榤Ⱦ盜ŭ飼蒱鄆 type: ""
currentReplicas: 99917513 currentReplicas: -212409426
currentRevision: "538" currentRevision: "538"
observedGeneration: 1124654959171263717 observedGeneration: 6696165662757041544
readyReplicas: 233229473 readyReplicas: -69450448
replicas: -1775998279 replicas: -629510776
updateRevision: "539" updateRevision: "539"
updatedReplicas: 421164481 updatedReplicas: 17761427

View File

@ -1693,26 +1693,30 @@
} }
}, },
"revisionHistoryLimit": -1166275743, "revisionHistoryLimit": -1166275743,
"minReadySeconds": -1056262432 "minReadySeconds": -1056262432,
"persistentVolumeClaimRetentionPolicy": {
"whenDeleted": "/Ş槀墺=Ĉ鳟/d",
"whenScaled": "á"
}
}, },
"status": { "status": {
"observedGeneration": -3770279213092072518, "observedGeneration": 5897193066236579633,
"replicas": -1669166259, "replicas": 1442750499,
"readyReplicas": -175399547, "readyReplicas": -1190434752,
"currentReplicas": 223338274, "currentReplicas": -1929177074,
"updatedReplicas": -1279136912, "updatedReplicas": -690843774,
"currentRevision": "538", "currentRevision": "538",
"updateRevision": "539", "updateRevision": "539",
"collisionCount": 1222237461, "collisionCount": 1329525670,
"conditions": [ "conditions": [
{ {
"type": "壣V礆á¤拈tY", "type": "\u0026嬜Š\u0026?鳢.ǀŭ瘢",
"status": "飼蒱鄆\u0026嬜Š\u0026?鳢.ǀŭ瘢颦z", "status": "Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6",
"lastTransitionTime": "2368-07-30T22:05:09Z", "lastTransitionTime": "2881-11-22T04:01:01Z",
"reason": "540", "reason": "540",
"message": "541" "message": "541"
} }
], ],
"availableReplicas": -1174483980 "availableReplicas": -1889018254
} }
} }

View File

@ -32,6 +32,9 @@ metadata:
uid: "7" uid: "7"
spec: spec:
minReadySeconds: -1056262432 minReadySeconds: -1056262432
persistentVolumeClaimRetentionPolicy:
whenDeleted: /Ş槀墺=Ĉ鳟/d
whenScaled: á
podManagementPolicy: Ȍ%ÿ¼璤ňɈ podManagementPolicy: Ȍ%ÿ¼璤ňɈ
replicas: 896585016 replicas: 896585016
revisionHistoryLimit: -1166275743 revisionHistoryLimit: -1166275743
@ -1164,18 +1167,18 @@ spec:
phase: ñƍU烈 źfjǰɪ phase: ñƍU烈 źfjǰɪ
resizeStatus: ʨɺC`牯 resizeStatus: ʨɺC`牯
status: status:
availableReplicas: -1174483980 availableReplicas: -1889018254
collisionCount: 1222237461 collisionCount: 1329525670
conditions: conditions:
- lastTransitionTime: "2368-07-30T22:05:09Z" - lastTransitionTime: "2881-11-22T04:01:01Z"
message: "541" message: "541"
reason: "540" reason: "540"
status: 飼蒱鄆&嬜Š&?鳢.ǀŭ瘢颦z status: Z氞唬蹵ɥeȿĦ`垨Džɞ堹ǖ*Oɑ埩6
type: 壣V礆á¤拈tY type: '&嬜Š&?鳢.ǀŭ瘢'
currentReplicas: 223338274 currentReplicas: -1929177074
currentRevision: "538" currentRevision: "538"
observedGeneration: -3770279213092072518 observedGeneration: 5897193066236579633
readyReplicas: -175399547 readyReplicas: -1190434752
replicas: -1669166259 replicas: 1442750499
updateRevision: "539" updateRevision: "539"
updatedReplicas: -1279136912 updatedReplicas: -690843774

View File

@ -0,0 +1,52 @@
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1
import (
v1 "k8s.io/api/apps/v1"
)
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use
// with apply.
type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct {
WhenDeleted *v1.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"`
WhenScaled *v1.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
}
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with
// apply.
func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
}
// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenDeleted field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenDeleted = &value
return b
}
// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenScaled field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenScaled = &value
return b
}

View File

@ -27,15 +27,16 @@ import (
// StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use
// with apply. // with apply.
type StatefulSetSpecApplyConfiguration struct { type StatefulSetSpecApplyConfiguration struct {
Replicas *int32 `json:"replicas,omitempty"` Replicas *int32 `json:"replicas,omitempty"`
Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"`
Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"`
VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"`
ServiceName *string `json:"serviceName,omitempty"` ServiceName *string `json:"serviceName,omitempty"`
PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"`
UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"`
RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
} }
// StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with
@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St
b.MinReadySeconds = &value b.MinReadySeconds = &value
return b return b
} }
// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call.
func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration {
b.PersistentVolumeClaimRetentionPolicy = value
return b
}

View File

@ -0,0 +1,52 @@
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1beta1
import (
v1beta1 "k8s.io/api/apps/v1beta1"
)
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use
// with apply.
type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct {
WhenDeleted *v1beta1.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"`
WhenScaled *v1beta1.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
}
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with
// apply.
func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
}
// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenDeleted field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1beta1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenDeleted = &value
return b
}
// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenScaled field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1beta1.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenScaled = &value
return b
}

View File

@ -27,15 +27,16 @@ import (
// StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use
// with apply. // with apply.
type StatefulSetSpecApplyConfiguration struct { type StatefulSetSpecApplyConfiguration struct {
Replicas *int32 `json:"replicas,omitempty"` Replicas *int32 `json:"replicas,omitempty"`
Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"`
Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"`
VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"`
ServiceName *string `json:"serviceName,omitempty"` ServiceName *string `json:"serviceName,omitempty"`
PodManagementPolicy *v1beta1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` PodManagementPolicy *v1beta1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"`
UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"`
RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
} }
// StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with
@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St
b.MinReadySeconds = &value b.MinReadySeconds = &value
return b return b
} }
// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call.
func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration {
b.PersistentVolumeClaimRetentionPolicy = value
return b
}

View File

@ -0,0 +1,52 @@
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1beta2
import (
v1beta2 "k8s.io/api/apps/v1beta2"
)
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration represents an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use
// with apply.
type StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration struct {
WhenDeleted *v1beta2.PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"`
WhenScaled *v1beta2.PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
}
// StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration constructs an declarative configuration of the StatefulSetPersistentVolumeClaimRetentionPolicy type for use with
// apply.
func StatefulSetPersistentVolumeClaimRetentionPolicy() *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
return &StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
}
// WithWhenDeleted sets the WhenDeleted field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenDeleted field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenDeleted(value v1beta2.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenDeleted = &value
return b
}
// WithWhenScaled sets the WhenScaled field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the WhenScaled field is set to the value of the last call.
func (b *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) WithWhenScaled(value v1beta2.PersistentVolumeClaimRetentionPolicyType) *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration {
b.WhenScaled = &value
return b
}

View File

@ -27,15 +27,16 @@ import (
// StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use // StatefulSetSpecApplyConfiguration represents an declarative configuration of the StatefulSetSpec type for use
// with apply. // with apply.
type StatefulSetSpecApplyConfiguration struct { type StatefulSetSpecApplyConfiguration struct {
Replicas *int32 `json:"replicas,omitempty"` Replicas *int32 `json:"replicas,omitempty"`
Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"`
Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"` Template *corev1.PodTemplateSpecApplyConfiguration `json:"template,omitempty"`
VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"` VolumeClaimTemplates []corev1.PersistentVolumeClaimApplyConfiguration `json:"volumeClaimTemplates,omitempty"`
ServiceName *string `json:"serviceName,omitempty"` ServiceName *string `json:"serviceName,omitempty"`
PodManagementPolicy *v1beta2.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` PodManagementPolicy *v1beta2.PodManagementPolicyType `json:"podManagementPolicy,omitempty"`
UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` UpdateStrategy *StatefulSetUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"`
RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
} }
// StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with // StatefulSetSpecApplyConfiguration constructs an declarative configuration of the StatefulSetSpec type for use with
@ -120,3 +121,11 @@ func (b *StatefulSetSpecApplyConfiguration) WithMinReadySeconds(value int32) *St
b.MinReadySeconds = &value b.MinReadySeconds = &value
return b return b
} }
// WithPersistentVolumeClaimRetentionPolicy sets the PersistentVolumeClaimRetentionPolicy field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the PersistentVolumeClaimRetentionPolicy field is set to the value of the last call.
func (b *StatefulSetSpecApplyConfiguration) WithPersistentVolumeClaimRetentionPolicy(value *StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration) *StatefulSetSpecApplyConfiguration {
b.PersistentVolumeClaimRetentionPolicy = value
return b
}

View File

@ -907,12 +907,24 @@ var schemaYAML = typed.YAMLObject(`types:
type: type:
scalar: string scalar: string
default: "" default: ""
- name: io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy
map:
fields:
- name: whenDeleted
type:
scalar: string
- name: whenScaled
type:
scalar: string
- name: io.k8s.api.apps.v1.StatefulSetSpec - name: io.k8s.api.apps.v1.StatefulSetSpec
map: map:
fields: fields:
- name: minReadySeconds - name: minReadySeconds
type: type:
scalar: numeric scalar: numeric
- name: persistentVolumeClaimRetentionPolicy
type:
namedType: io.k8s.api.apps.v1.StatefulSetPersistentVolumeClaimRetentionPolicy
- name: podManagementPolicy - name: podManagementPolicy
type: type:
scalar: string scalar: string
@ -1195,12 +1207,24 @@ var schemaYAML = typed.YAMLObject(`types:
type: type:
scalar: string scalar: string
default: "" default: ""
- name: io.k8s.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy
map:
fields:
- name: whenDeleted
type:
scalar: string
- name: whenScaled
type:
scalar: string
- name: io.k8s.api.apps.v1beta1.StatefulSetSpec - name: io.k8s.api.apps.v1beta1.StatefulSetSpec
map: map:
fields: fields:
- name: minReadySeconds - name: minReadySeconds
type: type:
scalar: numeric scalar: numeric
- name: persistentVolumeClaimRetentionPolicy
type:
namedType: io.k8s.api.apps.v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy
- name: podManagementPolicy - name: podManagementPolicy
type: type:
scalar: string scalar: string
@ -1681,12 +1705,24 @@ var schemaYAML = typed.YAMLObject(`types:
type: type:
scalar: string scalar: string
default: "" default: ""
- name: io.k8s.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy
map:
fields:
- name: whenDeleted
type:
scalar: string
- name: whenScaled
type:
scalar: string
- name: io.k8s.api.apps.v1beta2.StatefulSetSpec - name: io.k8s.api.apps.v1beta2.StatefulSetSpec
map: map:
fields: fields:
- name: minReadySeconds - name: minReadySeconds
type: type:
scalar: numeric scalar: numeric
- name: persistentVolumeClaimRetentionPolicy
type:
namedType: io.k8s.api.apps.v1beta2.StatefulSetPersistentVolumeClaimRetentionPolicy
- name: podManagementPolicy - name: podManagementPolicy
type: type:
scalar: string scalar: string

View File

@ -189,6 +189,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &applyconfigurationsappsv1.StatefulSetApplyConfiguration{} return &applyconfigurationsappsv1.StatefulSetApplyConfiguration{}
case appsv1.SchemeGroupVersion.WithKind("StatefulSetCondition"): case appsv1.SchemeGroupVersion.WithKind("StatefulSetCondition"):
return &applyconfigurationsappsv1.StatefulSetConditionApplyConfiguration{} return &applyconfigurationsappsv1.StatefulSetConditionApplyConfiguration{}
case appsv1.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"):
return &applyconfigurationsappsv1.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
case appsv1.SchemeGroupVersion.WithKind("StatefulSetSpec"): case appsv1.SchemeGroupVersion.WithKind("StatefulSetSpec"):
return &applyconfigurationsappsv1.StatefulSetSpecApplyConfiguration{} return &applyconfigurationsappsv1.StatefulSetSpecApplyConfiguration{}
case appsv1.SchemeGroupVersion.WithKind("StatefulSetStatus"): case appsv1.SchemeGroupVersion.WithKind("StatefulSetStatus"):
@ -219,6 +221,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &applyconfigurationsappsv1beta1.StatefulSetApplyConfiguration{} return &applyconfigurationsappsv1beta1.StatefulSetApplyConfiguration{}
case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetCondition"): case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetCondition"):
return &applyconfigurationsappsv1beta1.StatefulSetConditionApplyConfiguration{} return &applyconfigurationsappsv1beta1.StatefulSetConditionApplyConfiguration{}
case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"):
return &applyconfigurationsappsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetSpec"): case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetSpec"):
return &applyconfigurationsappsv1beta1.StatefulSetSpecApplyConfiguration{} return &applyconfigurationsappsv1beta1.StatefulSetSpecApplyConfiguration{}
case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetStatus"): case appsv1beta1.SchemeGroupVersion.WithKind("StatefulSetStatus"):
@ -269,6 +273,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &appsv1beta2.StatefulSetApplyConfiguration{} return &appsv1beta2.StatefulSetApplyConfiguration{}
case v1beta2.SchemeGroupVersion.WithKind("StatefulSetCondition"): case v1beta2.SchemeGroupVersion.WithKind("StatefulSetCondition"):
return &appsv1beta2.StatefulSetConditionApplyConfiguration{} return &appsv1beta2.StatefulSetConditionApplyConfiguration{}
case v1beta2.SchemeGroupVersion.WithKind("StatefulSetPersistentVolumeClaimRetentionPolicy"):
return &appsv1beta2.StatefulSetPersistentVolumeClaimRetentionPolicyApplyConfiguration{}
case v1beta2.SchemeGroupVersion.WithKind("StatefulSetSpec"): case v1beta2.SchemeGroupVersion.WithKind("StatefulSetSpec"):
return &appsv1beta2.StatefulSetSpecApplyConfiguration{} return &appsv1beta2.StatefulSetSpecApplyConfiguration{}
case v1beta2.SchemeGroupVersion.WithKind("StatefulSetStatus"): case v1beta2.SchemeGroupVersion.WithKind("StatefulSetStatus"):

View File

@ -20,6 +20,8 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"regexp"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -1142,6 +1144,7 @@ var _ = SIGDescribe("StatefulSet", func() {
appTester.run() appTester.run()
}) })
}) })
// Make sure minReadySeconds is honored // Make sure minReadySeconds is honored
// Don't mark it as conformance yet // Don't mark it as conformance yet
ginkgo.It("MinReadySeconds should be honored when enabled", func() { ginkgo.It("MinReadySeconds should be honored when enabled", func() {
@ -1202,6 +1205,144 @@ var _ = SIGDescribe("StatefulSet", func() {
framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 2, out) framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 2, out)
} }
}) })
ginkgo.Describe("Non-retain StatefulSetPersistentVolumeClaimPolicy [Feature:StatefulSetAutoDeletePVC]", func() {
ssName := "ss"
labels := map[string]string{
"foo": "bar",
"baz": "blah",
}
headlessSvcName := "test"
var statefulPodMounts, podMounts []v1.VolumeMount
var ss *appsv1.StatefulSet
ginkgo.BeforeEach(func() {
statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}}
podMounts = []v1.VolumeMount{{Name: "home", MountPath: "/home"}}
ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, statefulPodMounts, podMounts, labels)
ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns)
headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels)
_, err := c.CoreV1().Services(ns).Create(context.TODO(), headlessService, metav1.CreateOptions{})
framework.ExpectNoError(err)
})
ginkgo.AfterEach(func() {
if ginkgo.CurrentGinkgoTestDescription().Failed {
framework.DumpDebugInfo(c, ns)
}
framework.Logf("Deleting all statefulset in ns %v", ns)
e2estatefulset.DeleteAllStatefulSets(c, ns)
})
ginkgo.It("should delete PVCs with a WhenDeleted policy", func() {
e2epv.SkipIfNoDefaultStorageClass(c)
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
*(ss.Spec.Replicas) = 3
ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
}
_, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
framework.ExpectNoError(err)
ginkgo.By("Confirming all 3 PVCs exist with their owner refs")
err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false)
framework.ExpectNoError(err)
ginkgo.By("Deleting stateful set " + ss.Name)
err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{})
framework.ExpectNoError(err)
ginkgo.By("Verifying PVCs deleted")
err = verifyStatefulSetPVCsExist(c, ss, []int{})
framework.ExpectNoError(err)
})
ginkgo.It("should delete PVCs with a OnScaledown policy", func() {
e2epv.SkipIfNoDefaultStorageClass(c)
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
*(ss.Spec.Replicas) = 3
ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
}
_, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
framework.ExpectNoError(err)
ginkgo.By("Confirming all 3 PVCs exist")
err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2})
framework.ExpectNoError(err)
ginkgo.By("Scaling stateful set " + ss.Name + " to one replica")
ss, err = e2estatefulset.Scale(c, ss, 1)
framework.ExpectNoError(err)
ginkgo.By("Verifying all but one PVC deleted")
err = verifyStatefulSetPVCsExist(c, ss, []int{0})
framework.ExpectNoError(err)
})
ginkgo.It("should delete PVCs after adopting pod (WhenDeleted)", func() {
e2epv.SkipIfNoDefaultStorageClass(c)
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
*(ss.Spec.Replicas) = 3
ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
}
_, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
framework.ExpectNoError(err)
ginkgo.By("Confirming all 3 PVCs exist with their owner refs")
err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false)
framework.ExpectNoError(err)
ginkgo.By("Orphaning the 3rd pod")
patch, err := json.Marshal(metav1.ObjectMeta{
OwnerReferences: []metav1.OwnerReference{},
})
framework.ExpectNoError(err, "Could not Marshal JSON for patch payload")
_, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "")
framework.ExpectNoError(err, "Could not patch payload")
ginkgo.By("Deleting stateful set " + ss.Name)
err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{})
framework.ExpectNoError(err)
ginkgo.By("Verifying PVCs deleted")
err = verifyStatefulSetPVCsExist(c, ss, []int{})
framework.ExpectNoError(err)
})
ginkgo.It("should delete PVCs after adopting pod (WhenScaled) [Feature:StatefulSetAutoDeletePVC]", func() {
e2epv.SkipIfNoDefaultStorageClass(c)
ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
*(ss.Spec.Replicas) = 3
ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
}
_, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{})
framework.ExpectNoError(err)
ginkgo.By("Confirming all 3 PVCs exist")
err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2})
framework.ExpectNoError(err)
ginkgo.By("Orphaning the 3rd pod")
patch, err := json.Marshal(metav1.ObjectMeta{
OwnerReferences: []metav1.OwnerReference{},
})
framework.ExpectNoError(err, "Could not Marshal JSON for patch payload")
_, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "")
framework.ExpectNoError(err, "Could not patch payload")
ginkgo.By("Scaling stateful set " + ss.Name + " to one replica")
ss, err = e2estatefulset.Scale(c, ss, 1)
framework.ExpectNoError(err)
ginkgo.By("Verifying all but one PVC deleted")
err = verifyStatefulSetPVCsExist(c, ss, []int{0})
framework.ExpectNoError(err)
})
})
}) })
func kubectlExecWithRetries(ns string, args ...string) (out string) { func kubectlExecWithRetries(ns string, args ...string) (out string) {
@ -1645,3 +1786,111 @@ func getStatefulSet(c clientset.Interface, namespace, name string) *appsv1.State
} }
return ss return ss
} }
// verifyStatefulSetPVCsExist confirms that exactly the PVCs for ss with the specified ids exist. This polls until the situation occurs, an error happens, or until timeout (in the latter case an error is also returned). Beware that this cannot tell if a PVC will be deleted at some point in the future, so if used to confirm that no PVCs are deleted, the caller should wait for some event giving the PVCs a reasonable chance to be deleted, before calling this function.
func verifyStatefulSetPVCsExist(c clientset.Interface, ss *appsv1.StatefulSet, claimIds []int) error {
idSet := map[int]struct{}{}
for _, id := range claimIds {
idSet[id] = struct{}{}
}
return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) {
pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()})
if err != nil {
framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err)
return false, nil
}
for _, claim := range ss.Spec.VolumeClaimTemplates {
pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name))
seenPVCs := map[int]struct{}{}
for _, pvc := range pvcList.Items {
matches := pvcNameRE.FindStringSubmatch(pvc.Name)
if len(matches) != 2 {
continue
}
ordinal, err := strconv.ParseInt(matches[1], 10, 32)
if err != nil {
framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err)
return false, err
}
if _, found := idSet[int(ordinal)]; !found {
return false, nil // Retry until the PVCs are consistent.
} else {
seenPVCs[int(ordinal)] = struct{}{}
}
}
if len(seenPVCs) != len(idSet) {
framework.Logf("Found %d of %d PVCs", len(seenPVCs), len(idSet))
return false, nil // Retry until the PVCs are consistent.
}
}
return true, nil
})
}
// verifyStatefulSetPVCsExistWithOwnerRefs works as verifyStatefulSetPVCsExist, but also waits for the ownerRefs to match.
func verifyStatefulSetPVCsExistWithOwnerRefs(c clientset.Interface, ss *appsv1.StatefulSet, claimIndicies []int, wantSetRef, wantPodRef bool) error {
indexSet := map[int]struct{}{}
for _, id := range claimIndicies {
indexSet[id] = struct{}{}
}
set := getStatefulSet(c, ss.Namespace, ss.Name)
setUID := set.GetUID()
if setUID == "" {
framework.Failf("Statefulset %s mising UID", ss.Name)
}
return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) {
pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()})
if err != nil {
framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err)
return false, nil
}
for _, claim := range ss.Spec.VolumeClaimTemplates {
pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name))
seenPVCs := map[int]struct{}{}
for _, pvc := range pvcList.Items {
matches := pvcNameRE.FindStringSubmatch(pvc.Name)
if len(matches) != 2 {
continue
}
ordinal, err := strconv.ParseInt(matches[1], 10, 32)
if err != nil {
framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err)
return false, err
}
if _, found := indexSet[int(ordinal)]; !found {
framework.Logf("Unexpected, retrying")
return false, nil // Retry until the PVCs are consistent.
}
var foundSetRef, foundPodRef bool
for _, ref := range pvc.GetOwnerReferences() {
if ref.Kind == "StatefulSet" && ref.UID == setUID {
foundSetRef = true
}
if ref.Kind == "Pod" {
podName := fmt.Sprintf("%s-%d", ss.Name, ordinal)
pod, err := c.CoreV1().Pods(ss.Namespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
framework.Logf("Pod %s not found, retrying (%v)", podName, err)
return false, nil
}
podUID := pod.GetUID()
if podUID == "" {
framework.Failf("Pod %s is missing UID", pod.Name)
}
if ref.UID == podUID {
foundPodRef = true
}
}
}
if foundSetRef == wantSetRef && foundPodRef == wantPodRef {
seenPVCs[int(ordinal)] = struct{}{}
}
}
if len(seenPVCs) != len(indexSet) {
framework.Logf("Only %d PVCs, retrying", len(seenPVCs))
return false, nil // Retry until the PVCs are consistent.
}
}
return true, nil
})
}