v1.APIResource
@@ -7972,6 +8006,13 @@ Both these may change in the future. Incoming requests are matched against the h
v1beta1.AllowedHostPath array |
|
+
+allowedFlexVolumes |
+AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the "Volumes" field. |
+false |
+v1beta1.AllowedFlexVolume array |
+ |
+
diff --git a/hack/lib/version.sh b/hack/lib/version.sh
index 3dab71052af..0df161e37c6 100644
--- a/hack/lib/version.sh
+++ b/hack/lib/version.sh
@@ -22,6 +22,7 @@
# source code.
# KUBE_GIT_TREE_STATE - "clean" indicates no changes since the git commit id
# "dirty" indicates source code changes after the git commit id
+# "archive" indicates the tree was produced by 'git archive'
# KUBE_GIT_VERSION - "vX.Y" used to indicate the last release version.
# KUBE_GIT_MAJOR - The major part of the version
# KUBE_GIT_MINOR - The minor component of the version
@@ -40,7 +41,7 @@ kube::version::get_version_vars() {
# we likely don't have a git tree, but these magic values may be filled in.
if [[ '$Format:%%$' == "%" ]]; then
KUBE_GIT_COMMIT='$Format:%H$'
- KUBE_GIT_TREE_STATE="git archive"
+ KUBE_GIT_TREE_STATE="archive"
# When a 'git archive' is exported, the '$Format:%D$' below will look
# something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: '
# can be extracted from it.
diff --git a/pkg/apis/extensions/types.go b/pkg/apis/extensions/types.go
index 79694aae5c8..e36972846bc 100644
--- a/pkg/apis/extensions/types.go
+++ b/pkg/apis/extensions/types.go
@@ -860,6 +860,11 @@ type PodSecurityPolicySpec struct {
// AllowedHostPaths is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
AllowedHostPaths []AllowedHostPath
+ // AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
+ // Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
+ // is allowed in the "Volumes" field.
+ // +optional
+ AllowedFlexVolumes []AllowedFlexVolume
}
// AllowedHostPath defines the host volume conditions that will be enabled by a policy
@@ -923,6 +928,12 @@ var (
All FSType = "*"
)
+// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
+type AllowedFlexVolume struct {
+ // Driver is the name of the Flexvolume driver.
+ Driver string
+}
+
// SELinuxStrategyOptions defines the strategy type and any options used to create the strategy.
type SELinuxStrategyOptions struct {
// Rule is the strategy that will dictate the allowable labels that may be set.
diff --git a/pkg/apis/extensions/v1beta1/zz_generated.conversion.go b/pkg/apis/extensions/v1beta1/zz_generated.conversion.go
index 89fe74883c5..320ad17fb63 100644
--- a/pkg/apis/extensions/v1beta1/zz_generated.conversion.go
+++ b/pkg/apis/extensions/v1beta1/zz_generated.conversion.go
@@ -41,6 +41,8 @@ func init() {
// Public to allow building arbitrary schemes.
func RegisterConversions(scheme *runtime.Scheme) error {
return scheme.AddGeneratedConversionFuncs(
+ Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume,
+ Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume,
Convert_v1beta1_AllowedHostPath_To_extensions_AllowedHostPath,
Convert_extensions_AllowedHostPath_To_v1beta1_AllowedHostPath,
Convert_v1beta1_CustomMetricCurrentStatus_To_extensions_CustomMetricCurrentStatus,
@@ -140,6 +142,26 @@ func RegisterConversions(scheme *runtime.Scheme) error {
)
}
+func autoConvert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in *v1beta1.AllowedFlexVolume, out *extensions.AllowedFlexVolume, s conversion.Scope) error {
+ out.Driver = in.Driver
+ return nil
+}
+
+// Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume is an autogenerated conversion function.
+func Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in *v1beta1.AllowedFlexVolume, out *extensions.AllowedFlexVolume, s conversion.Scope) error {
+ return autoConvert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in, out, s)
+}
+
+func autoConvert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in *extensions.AllowedFlexVolume, out *v1beta1.AllowedFlexVolume, s conversion.Scope) error {
+ out.Driver = in.Driver
+ return nil
+}
+
+// Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume is an autogenerated conversion function.
+func Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in *extensions.AllowedFlexVolume, out *v1beta1.AllowedFlexVolume, s conversion.Scope) error {
+ return autoConvert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in, out, s)
+}
+
func autoConvert_v1beta1_AllowedHostPath_To_extensions_AllowedHostPath(in *v1beta1.AllowedHostPath, out *extensions.AllowedHostPath, s conversion.Scope) error {
out.PathPrefix = in.PathPrefix
return nil
@@ -1072,6 +1094,7 @@ func autoConvert_v1beta1_PodSecurityPolicySpec_To_extensions_PodSecurityPolicySp
return err
}
out.AllowedHostPaths = *(*[]extensions.AllowedHostPath)(unsafe.Pointer(&in.AllowedHostPaths))
+ out.AllowedFlexVolumes = *(*[]extensions.AllowedFlexVolume)(unsafe.Pointer(&in.AllowedFlexVolumes))
return nil
}
@@ -1108,6 +1131,7 @@ func autoConvert_extensions_PodSecurityPolicySpec_To_v1beta1_PodSecurityPolicySp
return err
}
out.AllowedHostPaths = *(*[]v1beta1.AllowedHostPath)(unsafe.Pointer(&in.AllowedHostPaths))
+ out.AllowedFlexVolumes = *(*[]v1beta1.AllowedFlexVolume)(unsafe.Pointer(&in.AllowedFlexVolumes))
return nil
}
diff --git a/pkg/apis/extensions/validation/validation.go b/pkg/apis/extensions/validation/validation.go
index bab3fd2f4a4..20cc279e8fd 100644
--- a/pkg/apis/extensions/validation/validation.go
+++ b/pkg/apis/extensions/validation/validation.go
@@ -655,6 +655,7 @@ func ValidatePodSecurityPolicySpec(spec *extensions.PodSecurityPolicySpec, fldPa
allErrs = append(allErrs, validatePSPCapsAgainstDrops(spec.RequiredDropCapabilities, spec.AllowedCapabilities, field.NewPath("allowedCapabilities"))...)
allErrs = append(allErrs, validatePSPDefaultAllowPrivilegeEscalation(fldPath.Child("defaultAllowPrivilegeEscalation"), spec.DefaultAllowPrivilegeEscalation, spec.AllowPrivilegeEscalation)...)
allErrs = append(allErrs, validatePSPAllowedHostPaths(fldPath.Child("allowedHostPaths"), spec.AllowedHostPaths)...)
+ allErrs = append(allErrs, validatePSPAllowedFlexVolumes(fldPath.Child("allowedFlexVolumes"), spec.AllowedFlexVolumes)...)
return allErrs
}
@@ -721,6 +722,20 @@ func validatePSPAllowedHostPaths(fldPath *field.Path, allowedHostPaths []extensi
return allErrs
}
+// validatePSPAllowedFlexVolumes
+func validatePSPAllowedFlexVolumes(fldPath *field.Path, flexVolumes []extensions.AllowedFlexVolume) field.ErrorList {
+ allErrs := field.ErrorList{}
+ if len(flexVolumes) > 0 {
+ for idx, fv := range flexVolumes {
+ if len(fv.Driver) == 0 {
+ allErrs = append(allErrs, field.Required(fldPath.Child("allowedFlexVolumes").Index(idx).Child("driver"),
+ "must specify a driver"))
+ }
+ }
+ }
+ return allErrs
+}
+
// validatePSPSELinux validates the SELinux fields of PodSecurityPolicy.
func validatePSPSELinux(fldPath *field.Path, seLinux *extensions.SELinuxStrategyOptions) field.ErrorList {
allErrs := field.ErrorList{}
@@ -802,7 +817,6 @@ func validatePodSecurityPolicyVolumes(fldPath *field.Path, volumes []extensions.
allErrs = append(allErrs, field.NotSupported(fldPath.Child("volumes"), v, allowed.List()))
}
}
-
return allErrs
}
diff --git a/pkg/apis/extensions/validation/validation_test.go b/pkg/apis/extensions/validation/validation_test.go
index 2a432da9b4a..4a3818801ff 100644
--- a/pkg/apis/extensions/validation/validation_test.go
+++ b/pkg/apis/extensions/validation/validation_test.go
@@ -2450,6 +2450,13 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
pe := true
invalidDefaultAllowPrivilegeEscalation.Spec.DefaultAllowPrivilegeEscalation = &pe
+ emptyFlexDriver := validPSP()
+ emptyFlexDriver.Spec.Volumes = []extensions.FSType{extensions.FlexVolume}
+ emptyFlexDriver.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{{}}
+
+ nonEmptyFlexVolumes := validPSP()
+ nonEmptyFlexVolumes.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{{Driver: "example/driver"}}
+
type testCase struct {
psp *extensions.PodSecurityPolicy
errorType field.ErrorType
@@ -2581,6 +2588,11 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
errorType: field.ErrorTypeInvalid,
errorDetail: "must not contain '..'",
},
+ "empty flex volume driver": {
+ psp: emptyFlexDriver,
+ errorType: field.ErrorTypeRequired,
+ errorDetail: "must specify a driver",
+ },
}
for k, v := range errorCases {
@@ -2660,6 +2672,17 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
validDefaultAllowPrivilegeEscalation.Spec.DefaultAllowPrivilegeEscalation = &pe
validDefaultAllowPrivilegeEscalation.Spec.AllowPrivilegeEscalation = true
+ flexvolumeWhenFlexVolumesAllowed := validPSP()
+ flexvolumeWhenFlexVolumesAllowed.Spec.Volumes = []extensions.FSType{extensions.FlexVolume}
+ flexvolumeWhenFlexVolumesAllowed.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{
+ {Driver: "example/driver1"},
+ }
+
+ flexvolumeWhenAllVolumesAllowed := validPSP()
+ flexvolumeWhenAllVolumesAllowed.Spec.Volumes = []extensions.FSType{extensions.All}
+ flexvolumeWhenAllVolumesAllowed.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{
+ {Driver: "example/driver2"},
+ }
successCases := map[string]struct {
psp *extensions.PodSecurityPolicy
}{
@@ -2690,6 +2713,12 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
"valid defaultAllowPrivilegeEscalation as true": {
psp: validDefaultAllowPrivilegeEscalation,
},
+ "allow white-listed flexVolume when flex volumes are allowed": {
+ psp: flexvolumeWhenFlexVolumesAllowed,
+ },
+ "allow white-listed flexVolume when all volumes are allowed": {
+ psp: flexvolumeWhenAllVolumesAllowed,
+ },
}
for k, v := range successCases {
diff --git a/pkg/apis/extensions/zz_generated.deepcopy.go b/pkg/apis/extensions/zz_generated.deepcopy.go
index 3d94576924a..ea801b3e8f5 100644
--- a/pkg/apis/extensions/zz_generated.deepcopy.go
+++ b/pkg/apis/extensions/zz_generated.deepcopy.go
@@ -26,6 +26,22 @@ import (
core "k8s.io/kubernetes/pkg/apis/core"
)
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AllowedFlexVolume) DeepCopyInto(out *AllowedFlexVolume) {
+ *out = *in
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedFlexVolume.
+func (in *AllowedFlexVolume) DeepCopy() *AllowedFlexVolume {
+ if in == nil {
+ return nil
+ }
+ out := new(AllowedFlexVolume)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedHostPath) DeepCopyInto(out *AllowedHostPath) {
*out = *in
@@ -919,6 +935,11 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
*out = make([]AllowedHostPath, len(*in))
copy(*out, *in)
}
+ if in.AllowedFlexVolumes != nil {
+ in, out := &in.AllowedFlexVolumes, &out.AllowedFlexVolumes
+ *out = make([]AllowedFlexVolume, len(*in))
+ copy(*out, *in)
+ }
return
}
diff --git a/pkg/cloudprovider/providers/aws/BUILD b/pkg/cloudprovider/providers/aws/BUILD
index 5936b00945d..9a4a6a76539 100644
--- a/pkg/cloudprovider/providers/aws/BUILD
+++ b/pkg/cloudprovider/providers/aws/BUILD
@@ -50,6 +50,7 @@ go_library(
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go
index 292210bacb1..c25b0f58ea2 100644
--- a/pkg/cloudprovider/providers/aws/aws.go
+++ b/pkg/cloudprovider/providers/aws/aws.go
@@ -48,6 +48,7 @@ import (
"path"
"k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
@@ -285,6 +286,10 @@ type EC2 interface {
// Delete an EBS volume
DeleteVolume(*ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error)
+ ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error)
+
+ DescribeVolumeModifications(*ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error)
+
DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error)
CreateSecurityGroup(*ec2.CreateSecurityGroupInput) (*ec2.CreateSecurityGroupOutput, error)
@@ -453,6 +458,9 @@ type Volumes interface {
// Check if disks specified in argument map are still attached to their respective nodes.
DisksAreAttached(map[types.NodeName][]KubernetesVolumeID) (map[types.NodeName]map[KubernetesVolumeID]bool, error)
+
+ // Expand the disk to new size
+ ResizeDisk(diskName KubernetesVolumeID, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error)
}
// InstanceGroups is an interface for managing cloud-managed instance groups / autoscaling instance groups
@@ -833,6 +841,36 @@ func (s *awsSdkEC2) DeleteVolume(request *ec2.DeleteVolumeInput) (*ec2.DeleteVol
return resp, err
}
+func (s *awsSdkEC2) ModifyVolume(request *ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) {
+ requestTime := time.Now()
+ resp, err := s.ec2.ModifyVolume(request)
+ timeTaken := time.Since(requestTime).Seconds()
+ recordAwsMetric("modify_volume", timeTaken, err)
+ return resp, err
+}
+
+func (s *awsSdkEC2) DescribeVolumeModifications(request *ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error) {
+ requestTime := time.Now()
+ results := []*ec2.VolumeModification{}
+ var nextToken *string
+ for {
+ resp, err := s.ec2.DescribeVolumesModifications(request)
+ if err != nil {
+ recordAwsMetric("describe_volume_modification", 0, err)
+ return nil, fmt.Errorf("error listing volume modifictions : %v", err)
+ }
+ results = append(results, resp.VolumesModifications...)
+ nextToken = resp.NextToken
+ if aws.StringValue(nextToken) == "" {
+ break
+ }
+ request.NextToken = nextToken
+ }
+ timeTaken := time.Since(requestTime).Seconds()
+ recordAwsMetric("describe_volume_modification", timeTaken, nil)
+ return results, nil
+}
+
func (s *awsSdkEC2) DescribeSubnets(request *ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) {
// Subnets are not paged
response, err := s.ec2.DescribeSubnets(request)
@@ -1653,6 +1691,65 @@ func (d *awsDisk) describeVolume() (*ec2.Volume, error) {
return volumes[0], nil
}
+func (d *awsDisk) describeVolumeModification() (*ec2.VolumeModification, error) {
+ volumeID := d.awsID
+ request := &ec2.DescribeVolumesModificationsInput{
+ VolumeIds: []*string{volumeID.awsString()},
+ }
+ volumeMods, err := d.ec2.DescribeVolumeModifications(request)
+
+ if err != nil {
+ return nil, fmt.Errorf("error describing volume modification %s with %v", volumeID, err)
+ }
+
+ if len(volumeMods) == 0 {
+ return nil, fmt.Errorf("no volume modifications found for %s", volumeID)
+ }
+ lastIndex := len(volumeMods) - 1
+ return volumeMods[lastIndex], nil
+}
+
+func (d *awsDisk) modifyVolume(requestGiB int64) (int64, error) {
+ volumeID := d.awsID
+
+ request := &ec2.ModifyVolumeInput{
+ VolumeId: volumeID.awsString(),
+ Size: aws.Int64(requestGiB),
+ }
+ output, err := d.ec2.ModifyVolume(request)
+ if err != nil {
+ modifyError := fmt.Errorf("AWS modifyVolume failed for %s with %v", volumeID, err)
+ return requestGiB, modifyError
+ }
+
+ volumeModification := output.VolumeModification
+
+ if aws.StringValue(volumeModification.ModificationState) == ec2.VolumeModificationStateCompleted {
+ return aws.Int64Value(volumeModification.TargetSize), nil
+ }
+
+ backoff := wait.Backoff{
+ Duration: 1 * time.Second,
+ Factor: 2,
+ Steps: 10,
+ }
+
+ checkForResize := func() (bool, error) {
+ volumeModification, err := d.describeVolumeModification()
+
+ if err != nil {
+ return false, err
+ }
+
+ if aws.StringValue(volumeModification.ModificationState) == ec2.VolumeModificationStateCompleted {
+ return true, nil
+ }
+ return false, nil
+ }
+ waitWithErr := wait.ExponentialBackoff(backoff, checkForResize)
+ return requestGiB, waitWithErr
+}
+
// applyUnSchedulableTaint applies a unschedulable taint to a node after verifying
// if node has become unusable because of volumes getting stuck in attaching state.
func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) {
@@ -2321,6 +2418,37 @@ func (c *Cloud) DisksAreAttached(nodeDisks map[types.NodeName][]KubernetesVolume
return attached, nil
}
+func (c *Cloud) ResizeDisk(
+ diskName KubernetesVolumeID,
+ oldSize resource.Quantity,
+ newSize resource.Quantity) (resource.Quantity, error) {
+ awsDisk, err := newAWSDisk(c, diskName)
+ if err != nil {
+ return oldSize, err
+ }
+
+ volumeInfo, err := awsDisk.describeVolume()
+ if err != nil {
+ descErr := fmt.Errorf("AWS.ResizeDisk Error describing volume %s with %v", diskName, err)
+ return oldSize, descErr
+ }
+ requestBytes := newSize.Value()
+ // AWS resizes in chunks of GiB (not GB)
+ requestGiB := volume.RoundUpSize(requestBytes, 1024*1024*1024)
+ newSizeQuant := resource.MustParse(fmt.Sprintf("%dGi", requestGiB))
+
+ // If disk already if of greater or equal size than requested we return
+ if aws.Int64Value(volumeInfo.Size) >= requestGiB {
+ return newSizeQuant, nil
+ }
+ _, err = awsDisk.modifyVolume(requestGiB)
+
+ if err != nil {
+ return oldSize, err
+ }
+ return newSizeQuant, nil
+}
+
// Gets the current load balancer state
func (c *Cloud) describeLoadBalancer(name string) (*elb.LoadBalancerDescription, error) {
request := &elb.DescribeLoadBalancersInput{}
diff --git a/pkg/cloudprovider/providers/aws/aws_fakes.go b/pkg/cloudprovider/providers/aws/aws_fakes.go
index 036968673e7..77436327230 100644
--- a/pkg/cloudprovider/providers/aws/aws_fakes.go
+++ b/pkg/cloudprovider/providers/aws/aws_fakes.go
@@ -203,6 +203,14 @@ func (ec2i *FakeEC2Impl) RevokeSecurityGroupIngress(*ec2.RevokeSecurityGroupIngr
panic("Not implemented")
}
+func (ec2i *FakeEC2Impl) DescribeVolumeModifications(*ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error) {
+ panic("Not implemented")
+}
+
+func (ec2i *FakeEC2Impl) ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) {
+ panic("Not implemented")
+}
+
func (ec2i *FakeEC2Impl) CreateSubnet(request *ec2.Subnet) (*ec2.CreateSubnetOutput, error) {
ec2i.Subnets = append(ec2i.Subnets, request)
response := &ec2.CreateSubnetOutput{
diff --git a/pkg/controller/BUILD b/pkg/controller/BUILD
index 287ff8d32bd..fe5a3adc085 100644
--- a/pkg/controller/BUILD
+++ b/pkg/controller/BUILD
@@ -133,6 +133,7 @@ filegroup(
"//pkg/controller/volume/events:all-srcs",
"//pkg/controller/volume/expand:all-srcs",
"//pkg/controller/volume/persistentvolume:all-srcs",
+ "//pkg/controller/volume/pvcprotection:all-srcs",
],
tags = ["automanaged"],
)
diff --git a/pkg/controller/node/ipam/BUILD b/pkg/controller/node/ipam/BUILD
index 31c1c34838c..1f0d52855b3 100644
--- a/pkg/controller/node/ipam/BUILD
+++ b/pkg/controller/node/ipam/BUILD
@@ -16,12 +16,15 @@ go_test(
importpath = "k8s.io/kubernetes/pkg/controller/node/ipam",
library = ":go_default_library",
deps = [
+ "//pkg/controller:go_default_library",
"//pkg/controller/node/ipam/cidrset:go_default_library",
"//pkg/controller/node/ipam/test:go_default_library",
"//pkg/controller/testutil:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+ "//vendor/k8s.io/client-go/informers:go_default_library",
+ "//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)
@@ -41,6 +44,7 @@ go_library(
deps = [
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
+ "//pkg/controller:go_default_library",
"//pkg/controller/node/ipam/cidrset:go_default_library",
"//pkg/controller/node/ipam/sync:go_default_library",
"//pkg/controller/node/util:go_default_library",
@@ -52,12 +56,14 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+ "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/scheme:go_default_library",
diff --git a/pkg/controller/node/ipam/cidr_allocator.go b/pkg/controller/node/ipam/cidr_allocator.go
index 56523009a0e..b9a97938ad3 100644
--- a/pkg/controller/node/ipam/cidr_allocator.go
+++ b/pkg/controller/node/ipam/cidr_allocator.go
@@ -80,12 +80,12 @@ type CIDRAllocator interface {
AllocateOrOccupyCIDR(node *v1.Node) error
// ReleaseCIDR releases the CIDR of the removed node
ReleaseCIDR(node *v1.Node) error
- // Register allocator with the nodeInformer for updates.
- Register(nodeInformer informers.NodeInformer)
+ // Run starts all the working logic of the allocator.
+ Run(stopCh <-chan struct{})
}
// New creates a new CIDR range allocator.
-func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, allocatorType CIDRAllocatorType, clusterCIDR, serviceCIDR *net.IPNet, nodeCIDRMaskSize int) (CIDRAllocator, error) {
+func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer, allocatorType CIDRAllocatorType, clusterCIDR, serviceCIDR *net.IPNet, nodeCIDRMaskSize int) (CIDRAllocator, error) {
nodeList, err := listNodes(kubeClient)
if err != nil {
return nil, err
@@ -93,9 +93,9 @@ func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, allocato
switch allocatorType {
case RangeAllocatorType:
- return NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList)
+ return NewCIDRRangeAllocator(kubeClient, nodeInformer, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList)
case CloudAllocatorType:
- return NewCloudCIDRAllocator(kubeClient, cloud)
+ return NewCloudCIDRAllocator(kubeClient, cloud, nodeInformer)
default:
return nil, fmt.Errorf("Invalid CIDR allocator type: %v", allocatorType)
}
diff --git a/pkg/controller/node/ipam/cloud_cidr_allocator.go b/pkg/controller/node/ipam/cloud_cidr_allocator.go
index 01502462071..1654c312532 100644
--- a/pkg/controller/node/ipam/cloud_cidr_allocator.go
+++ b/pkg/controller/node/ipam/cloud_cidr_allocator.go
@@ -25,9 +25,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
- "k8s.io/apimachinery/pkg/util/wait"
informers "k8s.io/client-go/informers/core/v1"
+ corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
@@ -37,6 +38,7 @@ import (
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+ "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/node/util"
nodeutil "k8s.io/kubernetes/pkg/util/node"
)
@@ -49,6 +51,12 @@ type cloudCIDRAllocator struct {
client clientset.Interface
cloud *gce.GCECloud
+ // nodeLister is able to list/get nodes and is populated by the shared informer passed to
+ // NewCloudCIDRAllocator.
+ nodeLister corelisters.NodeLister
+ // nodesSynced returns true if the node shared informer has been synced at least once.
+ nodesSynced cache.InformerSynced
+
// Channel that is used to pass updating Nodes to the background.
// This increases the throughput of CIDR assignment by parallelization
// and not blocking on long operations (which shouldn't be done from
@@ -64,7 +72,7 @@ type cloudCIDRAllocator struct {
var _ CIDRAllocator = (*cloudCIDRAllocator)(nil)
// NewCloudCIDRAllocator creates a new cloud CIDR allocator.
-func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Interface) (CIDRAllocator, error) {
+func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer) (CIDRAllocator, error) {
if client == nil {
glog.Fatalf("kubeClient is nil when starting NodeController")
}
@@ -84,20 +92,45 @@ func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Inter
ca := &cloudCIDRAllocator{
client: client,
cloud: gceCloud,
+ nodeLister: nodeInformer.Lister(),
+ nodesSynced: nodeInformer.Informer().HasSynced,
nodeUpdateChannel: make(chan string, cidrUpdateQueueSize),
recorder: recorder,
nodesInProcessing: sets.NewString(),
}
- for i := 0; i < cidrUpdateWorkers; i++ {
- // TODO: Take stopChan as an argument to NewCloudCIDRAllocator and pass it to the worker.
- go ca.worker(wait.NeverStop)
- }
+ nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: util.CreateAddNodeHandler(ca.AllocateOrOccupyCIDR),
+ UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
+ if newNode.Spec.PodCIDR == "" {
+ return ca.AllocateOrOccupyCIDR(newNode)
+ }
+ return nil
+ }),
+ DeleteFunc: util.CreateDeleteNodeHandler(ca.ReleaseCIDR),
+ })
glog.V(0).Infof("Using cloud CIDR allocator (provider: %v)", cloud.ProviderName())
return ca, nil
}
+func (ca *cloudCIDRAllocator) Run(stopCh <-chan struct{}) {
+ defer utilruntime.HandleCrash()
+
+ glog.Infof("Starting cloud CIDR allocator")
+ defer glog.Infof("Shutting down cloud CIDR allocator")
+
+ if !controller.WaitForCacheSync("cidrallocator", stopCh, ca.nodesSynced) {
+ return
+ }
+
+ for i := 0; i < cidrUpdateWorkers; i++ {
+ go ca.worker(stopCh)
+ }
+
+ <-stopCh
+}
+
func (ca *cloudCIDRAllocator) worker(stopChan <-chan struct{}) {
for {
select {
@@ -169,7 +202,7 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error {
for rep := 0; rep < cidrUpdateRetries; rep++ {
// TODO: change it to using PATCH instead of full Node updates.
- node, err = ca.client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+ node, err = ca.nodeLister.Get(nodeName)
if err != nil {
glog.Errorf("Failed while getting node %v to retry updating Node.Spec.PodCIDR: %v", nodeName, err)
continue
@@ -218,16 +251,3 @@ func (ca *cloudCIDRAllocator) ReleaseCIDR(node *v1.Node) error {
node.Name, node.Spec.PodCIDR)
return nil
}
-
-func (ca *cloudCIDRAllocator) Register(nodeInformer informers.NodeInformer) {
- nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- AddFunc: util.CreateAddNodeHandler(ca.AllocateOrOccupyCIDR),
- UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
- if newNode.Spec.PodCIDR == "" {
- return ca.AllocateOrOccupyCIDR(newNode)
- }
- return nil
- }),
- DeleteFunc: util.CreateDeleteNodeHandler(ca.ReleaseCIDR),
- })
-}
diff --git a/pkg/controller/node/ipam/range_allocator.go b/pkg/controller/node/ipam/range_allocator.go
index 102eac43d8e..13d1f77d7eb 100644
--- a/pkg/controller/node/ipam/range_allocator.go
+++ b/pkg/controller/node/ipam/range_allocator.go
@@ -25,16 +25,16 @@ import (
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
- "k8s.io/apimachinery/pkg/util/wait"
informers "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+ corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
-
+ "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/node/ipam/cidrset"
"k8s.io/kubernetes/pkg/controller/node/util"
)
@@ -45,6 +45,12 @@ type rangeAllocator struct {
clusterCIDR *net.IPNet
maxCIDRs int
+ // nodeLister is able to list/get nodes and is populated by the shared informer passed to
+ // NewCloudCIDRAllocator.
+ nodeLister corelisters.NodeLister
+ // nodesSynced returns true if the node shared informer has been synced at least once.
+ nodesSynced cache.InformerSynced
+
// Channel that is used to pass updating Nodes with assigned CIDRs to the background
// This increases a throughput of CIDR assignment by not blocking on long operations.
nodeCIDRUpdateChannel chan nodeAndCIDR
@@ -59,7 +65,7 @@ type rangeAllocator struct {
// Caller must ensure subNetMaskSize is not less than cluster CIDR mask size.
// Caller must always pass in a list of existing nodes so the new allocator
// can initialize its CIDR map. NodeList is only nil in testing.
-func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, subNetMaskSize int, nodeList *v1.NodeList) (CIDRAllocator, error) {
+func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.NodeInformer, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, subNetMaskSize int, nodeList *v1.NodeList) (CIDRAllocator, error) {
if client == nil {
glog.Fatalf("kubeClient is nil when starting NodeController")
}
@@ -78,6 +84,8 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s
client: client,
cidrs: set,
clusterCIDR: clusterCIDR,
+ nodeLister: nodeInformer.Lister(),
+ nodesSynced: nodeInformer.Informer().HasSynced,
nodeCIDRUpdateChannel: make(chan nodeAndCIDR, cidrUpdateQueueSize),
recorder: recorder,
nodesInProcessing: sets.NewString(),
@@ -107,14 +115,57 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s
}
}
}
- for i := 0; i < cidrUpdateWorkers; i++ {
- // TODO: Take stopChan as an argument to NewCIDRRangeAllocator and pass it to the worker.
- go ra.worker(wait.NeverStop)
- }
+
+ nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: util.CreateAddNodeHandler(ra.AllocateOrOccupyCIDR),
+ UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
+ // If the PodCIDR is not empty we either:
+ // - already processed a Node that already had a CIDR after NC restarted
+ // (cidr is marked as used),
+ // - already processed a Node successfully and allocated a CIDR for it
+ // (cidr is marked as used),
+ // - already processed a Node but we did saw a "timeout" response and
+ // request eventually got through in this case we haven't released
+ // the allocated CIDR (cidr is still marked as used).
+ // There's a possible error here:
+ // - NC sees a new Node and assigns a CIDR X to it,
+ // - Update Node call fails with a timeout,
+ // - Node is updated by some other component, NC sees an update and
+ // assigns CIDR Y to the Node,
+ // - Both CIDR X and CIDR Y are marked as used in the local cache,
+ // even though Node sees only CIDR Y
+ // The problem here is that in in-memory cache we see CIDR X as marked,
+ // which prevents it from being assigned to any new node. The cluster
+ // state is correct.
+ // Restart of NC fixes the issue.
+ if newNode.Spec.PodCIDR == "" {
+ return ra.AllocateOrOccupyCIDR(newNode)
+ }
+ return nil
+ }),
+ DeleteFunc: util.CreateDeleteNodeHandler(ra.ReleaseCIDR),
+ })
return ra, nil
}
+func (r *rangeAllocator) Run(stopCh <-chan struct{}) {
+ defer utilruntime.HandleCrash()
+
+ glog.Infof("Starting range CIDR allocator")
+ defer glog.Infof("Shutting down range CIDR allocator")
+
+ if !controller.WaitForCacheSync("cidrallocator", stopCh, r.nodesSynced) {
+ return
+ }
+
+ for i := 0; i < cidrUpdateWorkers; i++ {
+ go r.worker(stopCh)
+ }
+
+ <-stopCh
+}
+
func (r *rangeAllocator) worker(stopChan <-chan struct{}) {
for {
select {
@@ -232,7 +283,7 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
podCIDR := data.cidr.String()
for rep := 0; rep < cidrUpdateRetries; rep++ {
// TODO: change it to using PATCH instead of full Node updates.
- node, err = r.client.CoreV1().Nodes().Get(data.nodeName, metav1.GetOptions{})
+ node, err = r.nodeLister.Get(data.nodeName)
if err != nil {
glog.Errorf("Failed while getting node %v to retry updating Node.Spec.PodCIDR: %v", data.nodeName, err)
continue
@@ -269,35 +320,3 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
}
return err
}
-
-func (r *rangeAllocator) Register(nodeInformer informers.NodeInformer) {
- nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
- AddFunc: util.CreateAddNodeHandler(r.AllocateOrOccupyCIDR),
- UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
- // If the PodCIDR is not empty we either:
- // - already processed a Node that already had a CIDR after NC restarted
- // (cidr is marked as used),
- // - already processed a Node successfully and allocated a CIDR for it
- // (cidr is marked as used),
- // - already processed a Node but we did saw a "timeout" response and
- // request eventually got through in this case we haven't released
- // the allocated CIDR (cidr is still marked as used).
- // There's a possible error here:
- // - NC sees a new Node and assigns a CIDR X to it,
- // - Update Node call fails with a timeout,
- // - Node is updated by some other component, NC sees an update and
- // assigns CIDR Y to the Node,
- // - Both CIDR X and CIDR Y are marked as used in the local cache,
- // even though Node sees only CIDR Y
- // The problem here is that in in-memory cache we see CIDR X as marked,
- // which prevents it from being assigned to any new node. The cluster
- // state is correct.
- // Restart of NC fixes the issue.
- if newNode.Spec.PodCIDR == "" {
- return r.AllocateOrOccupyCIDR(newNode)
- }
- return nil
- }),
- DeleteFunc: util.CreateDeleteNodeHandler(r.ReleaseCIDR),
- })
-}
diff --git a/pkg/controller/node/ipam/range_allocator_test.go b/pkg/controller/node/ipam/range_allocator_test.go
index 3f911a899b3..c20ccd6a527 100644
--- a/pkg/controller/node/ipam/range_allocator_test.go
+++ b/pkg/controller/node/ipam/range_allocator_test.go
@@ -24,7 +24,10 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/client-go/informers"
+ coreinformers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes/fake"
+ "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/testutil"
)
@@ -32,6 +35,8 @@ const (
nodePollInterval = 100 * time.Millisecond
)
+var alwaysReady = func() bool { return true }
+
func waitForUpdatedNodeWithTimeout(nodeHandler *testutil.FakeNodeHandler, number int, timeout time.Duration) error {
return wait.Poll(nodePollInterval, timeout, func() (bool, error) {
if len(nodeHandler.GetUpdatedNodesCopy()) >= number {
@@ -41,6 +46,19 @@ func waitForUpdatedNodeWithTimeout(nodeHandler *testutil.FakeNodeHandler, number
})
}
+// Creates a fakeNodeInformer using the provided fakeNodeHandler.
+func getFakeNodeInformer(fakeNodeHandler *testutil.FakeNodeHandler) coreinformers.NodeInformer {
+ fakeClient := &fake.Clientset{}
+ fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, controller.NoResyncPeriodFunc())
+ fakeNodeInformer := fakeInformerFactory.Core().V1().Nodes()
+
+ for _, node := range fakeNodeHandler.Existing {
+ fakeNodeInformer.Informer().GetStore().Add(node)
+ }
+
+ return fakeNodeInformer
+}
+
func TestAllocateOrOccupyCIDRSuccess(t *testing.T) {
testCases := []struct {
description string
@@ -130,19 +148,23 @@ func TestAllocateOrOccupyCIDRSuccess(t *testing.T) {
expectedAllocatedCIDR string
allocatedCIDRs []string
}) {
- allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ // Initialize the range allocator.
+ allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ rangeAllocator, ok := allocator.(*rangeAllocator)
+ if !ok {
+ t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
+ return
+ }
+ rangeAllocator.nodesSynced = alwaysReady
+ rangeAllocator.recorder = testutil.NewFakeRecorder()
+ go allocator.Run(wait.NeverStop)
+
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
- rangeAllocator, ok := allocator.(*rangeAllocator)
- if !ok {
- t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
- return
- }
- rangeAllocator.recorder = testutil.NewFakeRecorder()
if err = rangeAllocator.cidrs.Occupy(cidr); err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
}
@@ -212,19 +234,23 @@ func TestAllocateOrOccupyCIDRFailure(t *testing.T) {
subNetMaskSize int
allocatedCIDRs []string
}) {
- allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ // Initialize the range allocator.
+ allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ rangeAllocator, ok := allocator.(*rangeAllocator)
+ if !ok {
+ t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
+ return
+ }
+ rangeAllocator.nodesSynced = alwaysReady
+ rangeAllocator.recorder = testutil.NewFakeRecorder()
+ go allocator.Run(wait.NeverStop)
+
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
- rangeAllocator, ok := allocator.(*rangeAllocator)
- if !ok {
- t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
- return
- }
- rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
@@ -324,19 +350,23 @@ func TestReleaseCIDRSuccess(t *testing.T) {
allocatedCIDRs []string
cidrsToRelease []string
}) {
- allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ // Initialize the range allocator.
+ allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
+ rangeAllocator, ok := allocator.(*rangeAllocator)
+ if !ok {
+ t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
+ return
+ }
+ rangeAllocator.nodesSynced = alwaysReady
+ rangeAllocator.recorder = testutil.NewFakeRecorder()
+ go allocator.Run(wait.NeverStop)
+
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
- rangeAllocator, ok := allocator.(*rangeAllocator)
- if !ok {
- t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
- return
- }
- rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go
index 8659a4327a3..ff77e6c7d5c 100644
--- a/pkg/controller/node/node_controller.go
+++ b/pkg/controller/node/node_controller.go
@@ -360,11 +360,10 @@ func NewNodeController(
} else {
var err error
nc.cidrAllocator, err = ipam.New(
- kubeClient, cloud, nc.allocatorType, nc.clusterCIDR, nc.serviceCIDR, nodeCIDRMaskSize)
+ kubeClient, cloud, nodeInformer, nc.allocatorType, nc.clusterCIDR, nc.serviceCIDR, nodeCIDRMaskSize)
if err != nil {
return nil, err
}
- nc.cidrAllocator.Register(nodeInformer)
}
}
@@ -585,6 +584,12 @@ func (nc *Controller) Run(stopCh <-chan struct{}) {
go wait.Until(nc.doEvictionPass, scheduler.NodeEvictionPeriod, wait.NeverStop)
}
+ if nc.allocateNodeCIDRs {
+ if nc.allocatorType != ipam.IPAMFromClusterAllocatorType && nc.allocatorType != ipam.IPAMFromCloudAllocatorType {
+ go nc.cidrAllocator.Run(wait.NeverStop)
+ }
+ }
+
<-stopCh
}
diff --git a/pkg/controller/volume/pvcprotection/BUILD b/pkg/controller/volume/pvcprotection/BUILD
new file mode 100644
index 00000000000..5c713a259d2
--- /dev/null
+++ b/pkg/controller/volume/pvcprotection/BUILD
@@ -0,0 +1,61 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["pvc_protection_controller.go"],
+ importpath = "k8s.io/kubernetes/pkg/controller/volume/pvcprotection",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//pkg/controller:go_default_library",
+ "//pkg/util/metrics:go_default_library",
+ "//pkg/volume/util:go_default_library",
+ "//pkg/volume/util/volumehelper:go_default_library",
+ "//vendor/github.com/golang/glog:go_default_library",
+ "//vendor/k8s.io/api/core/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+ "//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
+ "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+ "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+ "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+ "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+ ],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["pvc_protection_controller_test.go"],
+ importpath = "k8s.io/kubernetes/pkg/controller/volume/pvcprotection",
+ library = ":go_default_library",
+ deps = [
+ "//pkg/controller:go_default_library",
+ "//pkg/volume/util:go_default_library",
+ "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+ "//vendor/github.com/golang/glog:go_default_library",
+ "//vendor/k8s.io/api/core/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+ "//vendor/k8s.io/client-go/informers:go_default_library",
+ "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+ "//vendor/k8s.io/client-go/testing:go_default_library",
+ ],
+)
+
+filegroup(
+ name = "package-srcs",
+ srcs = glob(["**"]),
+ tags = ["automanaged"],
+ visibility = ["//visibility:private"],
+)
+
+filegroup(
+ name = "all-srcs",
+ srcs = [":package-srcs"],
+ tags = ["automanaged"],
+ visibility = ["//visibility:public"],
+)
diff --git a/pkg/controller/volume/pvcprotection/pvc_protection_controller.go b/pkg/controller/volume/pvcprotection/pvc_protection_controller.go
new file mode 100644
index 00000000000..40bf3e5de5c
--- /dev/null
+++ b/pkg/controller/volume/pvcprotection/pvc_protection_controller.go
@@ -0,0 +1,284 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvcprotection
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/golang/glog"
+ "k8s.io/api/core/v1"
+ apierrs "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/labels"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ "k8s.io/apimachinery/pkg/util/wait"
+ coreinformers "k8s.io/client-go/informers/core/v1"
+ clientset "k8s.io/client-go/kubernetes"
+ corelisters "k8s.io/client-go/listers/core/v1"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/client-go/util/workqueue"
+ "k8s.io/kubernetes/pkg/controller"
+ "k8s.io/kubernetes/pkg/util/metrics"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
+ "k8s.io/kubernetes/pkg/volume/util/volumehelper"
+)
+
+// Controller is controller that removes PVCProtectionFinalizer
+// from PVCs that are used by no pods.
+type Controller struct {
+ client clientset.Interface
+
+ pvcLister corelisters.PersistentVolumeClaimLister
+ pvcListerSynced cache.InformerSynced
+
+ podLister corelisters.PodLister
+ podListerSynced cache.InformerSynced
+
+ queue workqueue.RateLimitingInterface
+}
+
+// NewPVCProtectionController returns a new *{VCProtectionController.
+func NewPVCProtectionController(pvcInformer coreinformers.PersistentVolumeClaimInformer, podInformer coreinformers.PodInformer, cl clientset.Interface) *Controller {
+ e := &Controller{
+ client: cl,
+ queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcprotection"),
+ }
+ if cl != nil && cl.CoreV1().RESTClient().GetRateLimiter() != nil {
+ metrics.RegisterMetricAndTrackRateLimiterUsage("persistentvolumeclaim_protection_controller", cl.CoreV1().RESTClient().GetRateLimiter())
+ }
+
+ e.pvcLister = pvcInformer.Lister()
+ e.pvcListerSynced = pvcInformer.Informer().HasSynced
+ pvcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: e.pvcAddedUpdated,
+ UpdateFunc: func(old, new interface{}) {
+ e.pvcAddedUpdated(new)
+ },
+ })
+
+ e.podLister = podInformer.Lister()
+ e.podListerSynced = podInformer.Informer().HasSynced
+ podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+ AddFunc: func(obj interface{}) {
+ e.podAddedDeletedUpdated(obj, false)
+ },
+ DeleteFunc: func(obj interface{}) {
+ e.podAddedDeletedUpdated(obj, true)
+ },
+ UpdateFunc: func(old, new interface{}) {
+ e.podAddedDeletedUpdated(new, false)
+ },
+ })
+
+ return e
+}
+
+// Run runs the controller goroutines.
+func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
+ defer utilruntime.HandleCrash()
+ defer c.queue.ShutDown()
+
+ glog.Infof("Starting PVC protection controller")
+ defer glog.Infof("Shutting down PVC protection controller")
+
+ if !controller.WaitForCacheSync("PVC protection", stopCh, c.pvcListerSynced, c.podListerSynced) {
+ return
+ }
+
+ for i := 0; i < workers; i++ {
+ go wait.Until(c.runWorker, time.Second, stopCh)
+ }
+
+ <-stopCh
+}
+
+func (c *Controller) runWorker() {
+ for c.processNextWorkItem() {
+ }
+}
+
+// processNextWorkItem deals with one pvcKey off the queue. It returns false when it's time to quit.
+func (c *Controller) processNextWorkItem() bool {
+ pvcKey, quit := c.queue.Get()
+ if quit {
+ return false
+ }
+ defer c.queue.Done(pvcKey)
+
+ pvcNamespace, pvcName, err := cache.SplitMetaNamespaceKey(pvcKey.(string))
+ if err != nil {
+ utilruntime.HandleError(fmt.Errorf("Error parsing PVC key %q: %v", pvcKey, err))
+ return true
+ }
+
+ err = c.processPVC(pvcNamespace, pvcName)
+ if err == nil {
+ c.queue.Forget(pvcKey)
+ return true
+ }
+
+ utilruntime.HandleError(fmt.Errorf("PVC %v failed with : %v", pvcKey, err))
+ c.queue.AddRateLimited(pvcKey)
+
+ return true
+}
+
+func (c *Controller) processPVC(pvcNamespace, pvcName string) error {
+ glog.V(4).Infof("Processing PVC %s/%s", pvcNamespace, pvcName)
+ startTime := time.Now()
+ defer func() {
+ glog.V(4).Infof("Finished processing PVC %s/%s (%v)", pvcNamespace, pvcName, time.Now().Sub(startTime))
+ }()
+
+ pvc, err := c.pvcLister.PersistentVolumeClaims(pvcNamespace).Get(pvcName)
+ if apierrs.IsNotFound(err) {
+ glog.V(4).Infof("PVC %s/%s not found, ignoring", pvcNamespace, pvcName)
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+
+ if volumeutil.IsPVCBeingDeleted(pvc) && volumeutil.IsProtectionFinalizerPresent(pvc) {
+ // PVC should be deleted. Check if it's used and remove finalizer if
+ // it's not.
+ isUsed, err := c.isBeingUsed(pvc)
+ if err != nil {
+ return err
+ }
+ if !isUsed {
+ return c.removeFinalizer(pvc)
+ }
+ }
+
+ if !volumeutil.IsPVCBeingDeleted(pvc) && !volumeutil.IsProtectionFinalizerPresent(pvc) {
+ // PVC is not being deleted -> it should have the finalizer. The
+ // finalizer should be added by admission plugin, this is just to add
+ // the finalizer to old PVCs that were created before the admission
+ // plugin was enabled.
+ return c.addFinalizer(pvc)
+ }
+ return nil
+}
+
+func (c *Controller) addFinalizer(pvc *v1.PersistentVolumeClaim) error {
+ claimClone := pvc.DeepCopy()
+ volumeutil.AddProtectionFinalizer(claimClone)
+ _, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone)
+ if err != nil {
+ glog.V(3).Infof("Error adding protection finalizer to PVC %s/%s: %v", pvc.Namespace, pvc.Name)
+ return err
+ }
+ glog.V(3).Infof("Added protection finalizer to PVC %s/%s", pvc.Namespace, pvc.Name)
+ return nil
+}
+
+func (c *Controller) removeFinalizer(pvc *v1.PersistentVolumeClaim) error {
+ claimClone := pvc.DeepCopy()
+ volumeutil.RemoveProtectionFinalizer(claimClone)
+ _, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone)
+ if err != nil {
+ glog.V(3).Infof("Error removing protection finalizer from PVC %s/%s: %v", pvc.Namespace, pvc.Name, err)
+ return err
+ }
+ glog.V(3).Infof("Removed protection finalizer from PVC %s/%s", pvc.Namespace, pvc.Name)
+ return nil
+}
+
+func (c *Controller) isBeingUsed(pvc *v1.PersistentVolumeClaim) (bool, error) {
+ pods, err := c.podLister.Pods(pvc.Namespace).List(labels.Everything())
+ if err != nil {
+ return false, err
+ }
+ for _, pod := range pods {
+ if pod.Spec.NodeName == "" {
+ // This pod is not scheduled. We have a predicated in scheduler that
+ // prevents scheduling pods with deletion timestamp, so we can be
+ // pretty sure it won't be scheduled in parallel to this check.
+ // Therefore this pod does not block the PVC from deletion.
+ glog.V(4).Infof("Skipping unscheduled pod %s when checking PVC %s/%s", pod.Name, pvc.Namespace, pvc.Name)
+ continue
+ }
+ if volumehelper.IsPodTerminated(pod, pod.Status) {
+ // This pod is being unmounted/detached or is already
+ // unmounted/detached. It does not block the PVC from deletion.
+ continue
+ }
+ for _, volume := range pod.Spec.Volumes {
+ if volume.PersistentVolumeClaim == nil {
+ continue
+ }
+ if volume.PersistentVolumeClaim.ClaimName == pvc.Name {
+ glog.V(2).Infof("Keeping PVC %s/%s, it is used by pod %s/%s", pvc.Namespace, pvc.Name, pod.Namespace, pod.Name)
+ return true, nil
+ }
+ }
+ }
+
+ glog.V(3).Infof("PVC %s/%s is unused", pvc.Namespace, pvc.Name)
+ return false, nil
+}
+
+// pvcAddedUpdated reacts to pvc added/updated/deleted events
+func (c *Controller) pvcAddedUpdated(obj interface{}) {
+ pvc, ok := obj.(*v1.PersistentVolumeClaim)
+ if !ok {
+ utilruntime.HandleError(fmt.Errorf("PVC informer returned non-PVC object: %#v", obj))
+ return
+ }
+ key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pvc)
+ if err != nil {
+ utilruntime.HandleError(fmt.Errorf("Couldn't get key for Persistent Volume Claim %#v: %v", pvc, err))
+ return
+ }
+ glog.V(4).Infof("Got event on PVC %s", key)
+
+ if (!volumeutil.IsPVCBeingDeleted(pvc) && !volumeutil.IsProtectionFinalizerPresent(pvc)) || (volumeutil.IsPVCBeingDeleted(pvc) && volumeutil.IsProtectionFinalizerPresent(pvc)) {
+ c.queue.Add(key)
+ }
+}
+
+// podAddedDeletedUpdated reacts to Pod events
+func (c *Controller) podAddedDeletedUpdated(obj interface{}, deleted bool) {
+ pod, ok := obj.(*v1.Pod)
+ if !ok {
+ tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+ if !ok {
+ utilruntime.HandleError(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
+ return
+ }
+ pod, ok = tombstone.Obj.(*v1.Pod)
+ if !ok {
+ utilruntime.HandleError(fmt.Errorf("Tombstone contained object that is not a Pod %#v", obj))
+ return
+ }
+ }
+
+ // Filter out pods that can't help us to remove a finalizer on PVC
+ if !deleted && !volumehelper.IsPodTerminated(pod, pod.Status) && pod.Spec.NodeName != "" {
+ return
+ }
+
+ glog.V(4).Infof("Got event on pod %s/%s", pod.Namespace, pod.Name)
+
+ // Enqueue all PVCs that the pod uses
+ for _, volume := range pod.Spec.Volumes {
+ if volume.PersistentVolumeClaim != nil {
+ c.queue.Add(pod.Namespace + "/" + volume.PersistentVolumeClaim.ClaimName)
+ }
+ }
+}
diff --git a/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go b/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go
new file mode 100644
index 00000000000..0d7a2f9302c
--- /dev/null
+++ b/pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go
@@ -0,0 +1,397 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvcprotection
+
+import (
+ "errors"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/davecgh/go-spew/spew"
+ "github.com/golang/glog"
+
+ "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/client-go/informers"
+ "k8s.io/client-go/kubernetes/fake"
+ clienttesting "k8s.io/client-go/testing"
+ "k8s.io/kubernetes/pkg/controller"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
+)
+
+type reaction struct {
+ verb string
+ resource string
+ reactorfn clienttesting.ReactionFunc
+}
+
+const (
+ defaultNS = "default"
+ defaultPVCName = "pvc1"
+ defaultPodName = "pod1"
+ defaultNodeName = "node1"
+)
+
+func pod() *v1.Pod {
+ return &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: defaultPodName,
+ Namespace: defaultNS,
+ },
+ Spec: v1.PodSpec{
+ NodeName: defaultNodeName,
+ },
+ Status: v1.PodStatus{
+ Phase: v1.PodPending,
+ },
+ }
+}
+
+func unscheduled(pod *v1.Pod) *v1.Pod {
+ pod.Spec.NodeName = ""
+ return pod
+}
+
+func withPVC(pvcName string, pod *v1.Pod) *v1.Pod {
+ volume := v1.Volume{
+ Name: pvcName,
+ VolumeSource: v1.VolumeSource{
+ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+ ClaimName: pvcName,
+ },
+ },
+ }
+ pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+ return pod
+}
+
+func withEmptyDir(pod *v1.Pod) *v1.Pod {
+ volume := v1.Volume{
+ Name: "emptyDir",
+ VolumeSource: v1.VolumeSource{
+ EmptyDir: &v1.EmptyDirVolumeSource{},
+ },
+ }
+ pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+ return pod
+}
+
+func withStatus(phase v1.PodPhase, pod *v1.Pod) *v1.Pod {
+ pod.Status.Phase = phase
+ return pod
+}
+
+func pvc() *v1.PersistentVolumeClaim {
+ return &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: defaultPVCName,
+ Namespace: defaultNS,
+ },
+ }
+}
+
+func withProtectionFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
+ pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer)
+ return pvc
+}
+
+func deleted(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
+ pvc.DeletionTimestamp = &metav1.Time{}
+ return pvc
+}
+
+func generateUpdateErrorFunc(t *testing.T, failures int) clienttesting.ReactionFunc {
+ i := 0
+ return func(action clienttesting.Action) (bool, runtime.Object, error) {
+ i++
+ if i <= failures {
+ // Update fails
+ update, ok := action.(clienttesting.UpdateAction)
+
+ if !ok {
+ t.Fatalf("Reactor got non-update action: %+v", action)
+ }
+ acc, _ := meta.Accessor(update.GetObject())
+ return true, nil, apierrors.NewForbidden(update.GetResource().GroupResource(), acc.GetName(), errors.New("Mock error"))
+ }
+ // Update succeeds
+ return false, nil, nil
+ }
+}
+
+func TestPVCProtectionController(t *testing.T) {
+ pvcVer := schema.GroupVersionResource{
+ Group: v1.GroupName,
+ Version: "v1",
+ Resource: "persistentvolumeclaims",
+ }
+
+ tests := []struct {
+ name string
+ // Object to insert into fake kubeclient before the test starts.
+ initialObjects []runtime.Object
+ // Optional client reactors.
+ reactors []reaction
+ // PVC event to simulate. This PVC will be automatically added to
+ // initalObjects.
+ updatedPVC *v1.PersistentVolumeClaim
+ // Pod event to simulate. This Pod will be automatically added to
+ // initalObjects.
+ updatedPod *v1.Pod
+ // Pod event to similate. This Pod is *not* added to
+ // initalObjects.
+ deletedPod *v1.Pod
+ // List of expected kubeclient actions that should happen during the
+ // test.
+ expectedActions []clienttesting.Action
+ }{
+ //
+ // PVC events
+ //
+ {
+ name: "PVC without finalizer -> finalizer is added",
+ updatedPVC: pvc(),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
+ },
+ },
+ {
+ name: "PVC with finalizer -> no action",
+ updatedPVC: withProtectionFinalizer(pvc()),
+ expectedActions: []clienttesting.Action{},
+ },
+ {
+ name: "saving PVC finalizer fails -> controller retries",
+ updatedPVC: pvc(),
+ reactors: []reaction{
+ {
+ verb: "update",
+ resource: "persistentvolumeclaims",
+ reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/),
+ },
+ },
+ expectedActions: []clienttesting.Action{
+ // This fails
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
+ // This fails too
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
+ // This succeeds
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
+ },
+ },
+ {
+ name: "deleted PVC with finalizer -> finalizer is removed",
+ updatedPVC: deleted(withProtectionFinalizer(pvc())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ {
+ name: "finalizer removal fails -> controller retries",
+ updatedPVC: deleted(withProtectionFinalizer(pvc())),
+ reactors: []reaction{
+ {
+ verb: "update",
+ resource: "persistentvolumeclaims",
+ reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/),
+ },
+ },
+ expectedActions: []clienttesting.Action{
+ // Fails
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ // Fails too
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ // Succeeds
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ {
+ name: "deleted PVC with finalizer + pods with the PVC exists -> finalizer is not removed",
+ initialObjects: []runtime.Object{
+ withPVC(defaultPVCName, pod()),
+ },
+ updatedPVC: deleted(withProtectionFinalizer(pvc())),
+ expectedActions: []clienttesting.Action{},
+ },
+ {
+ name: "deleted PVC with finalizer + pods with unrelated PVC and EmptyDir exists -> finalizer is removed",
+ initialObjects: []runtime.Object{
+ withEmptyDir(withPVC("unrelatedPVC", pod())),
+ },
+ updatedPVC: deleted(withProtectionFinalizer(pvc())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ {
+ name: "deleted PVC with finalizer + pods with the PVC andis finished -> finalizer is removed",
+ initialObjects: []runtime.Object{
+ withStatus(v1.PodFailed, withPVC(defaultPVCName, pod())),
+ },
+ updatedPVC: deleted(withProtectionFinalizer(pvc())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ //
+ // Pod events
+ //
+ {
+ name: "updated running Pod -> no action",
+ initialObjects: []runtime.Object{
+ deleted(withProtectionFinalizer(pvc())),
+ },
+ updatedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())),
+ expectedActions: []clienttesting.Action{},
+ },
+ {
+ name: "updated finished Pod -> finalizer is removed",
+ initialObjects: []runtime.Object{
+ deleted(withProtectionFinalizer(pvc())),
+ },
+ updatedPod: withStatus(v1.PodSucceeded, withPVC(defaultPVCName, pod())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ {
+ name: "updated unscheduled Pod -> finalizer is removed",
+ initialObjects: []runtime.Object{
+ deleted(withProtectionFinalizer(pvc())),
+ },
+ updatedPod: unscheduled(withPVC(defaultPVCName, pod())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ {
+ name: "deleted running Pod -> finalizer is removed",
+ initialObjects: []runtime.Object{
+ deleted(withProtectionFinalizer(pvc())),
+ },
+ deletedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())),
+ expectedActions: []clienttesting.Action{
+ clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
+ },
+ },
+ }
+
+ for _, test := range tests {
+ // Create client with initial data
+ objs := test.initialObjects
+ if test.updatedPVC != nil {
+ objs = append(objs, test.updatedPVC)
+ }
+ if test.updatedPod != nil {
+ objs = append(objs, test.updatedPod)
+ }
+ client := fake.NewSimpleClientset(objs...)
+
+ // Create informers
+ informers := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
+ pvcInformer := informers.Core().V1().PersistentVolumeClaims()
+ podInformer := informers.Core().V1().Pods()
+
+ // Populate the informers with initial objects so the controller can
+ // Get() and List() it.
+ for _, obj := range objs {
+ switch obj.(type) {
+ case *v1.PersistentVolumeClaim:
+ pvcInformer.Informer().GetStore().Add(obj)
+ case *v1.Pod:
+ podInformer.Informer().GetStore().Add(obj)
+ default:
+ t.Fatalf("Unknown initalObject type: %+v", obj)
+ }
+ }
+
+ // Add reactor to inject test errors.
+ for _, reactor := range test.reactors {
+ client.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorfn)
+ }
+
+ // Create the controller
+ ctrl := NewPVCProtectionController(pvcInformer, podInformer, client)
+
+ // Start the test by simulating an event
+ if test.updatedPVC != nil {
+ ctrl.pvcAddedUpdated(test.updatedPVC)
+ }
+ if test.updatedPod != nil {
+ ctrl.podAddedDeletedUpdated(test.updatedPod, false)
+ }
+ if test.deletedPod != nil {
+ ctrl.podAddedDeletedUpdated(test.deletedPod, true)
+ }
+
+ // Process the controller queue until we get expected results
+ timeout := time.Now().Add(10 * time.Second)
+ lastReportedActionCount := 0
+ for {
+ if time.Now().After(timeout) {
+ t.Errorf("Test %q: timed out", test.name)
+ break
+ }
+ if ctrl.queue.Len() > 0 {
+ glog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len())
+ ctrl.processNextWorkItem()
+ }
+ if ctrl.queue.Len() > 0 {
+ // There is still some work in the queue, process it now
+ continue
+ }
+ currentActionCount := len(client.Actions())
+ if currentActionCount < len(test.expectedActions) {
+ // Do not log evey wait, only when the action count changes.
+ if lastReportedActionCount < currentActionCount {
+ glog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions))
+ lastReportedActionCount = currentActionCount
+ }
+ // The test expected more to happen, wait for the actions.
+ // Most probably it's exponential backoff
+ time.Sleep(10 * time.Millisecond)
+ continue
+ }
+ break
+ }
+ actions := client.Actions()
+ for i, action := range actions {
+ if len(test.expectedActions) < i+1 {
+ t.Errorf("Test %q: %d unexpected actions: %+v", test.name, len(actions)-len(test.expectedActions), spew.Sdump(actions[i:]))
+ break
+ }
+
+ expectedAction := test.expectedActions[i]
+ if !reflect.DeepEqual(expectedAction, action) {
+ t.Errorf("Test %q: action %d\nExpected:\n%s\ngot:\n%s", test.name, i, spew.Sdump(expectedAction), spew.Sdump(action))
+ }
+ }
+
+ if len(test.expectedActions) > len(actions) {
+ t.Errorf("Test %q: %d additional expected actions", test.name, len(test.expectedActions)-len(actions))
+ for _, a := range test.expectedActions[len(actions):] {
+ t.Logf(" %+v", a)
+ }
+ }
+
+ }
+}
diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
index d9bd748dc98..1ed30a62fda 100644
--- a/pkg/features/kube_features.go
+++ b/pkg/features/kube_features.go
@@ -200,6 +200,12 @@ const (
//
// Enable Block volume support in containers.
BlockVolume utilfeature.Feature = "BlockVolume"
+
+ // owner: @pospispa
+ //
+ // alpha: v1.9
+ // Postpone deletion of a persistent volume claim in case it is used by a pod
+ PVCProtection utilfeature.Feature = "PVCProtection"
)
func init() {
@@ -237,6 +243,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
CSIPersistentVolume: {Default: false, PreRelease: utilfeature.Alpha},
CustomPodDNS: {Default: false, PreRelease: utilfeature.Alpha},
BlockVolume: {Default: false, PreRelease: utilfeature.Alpha},
+ PVCProtection: {Default: false, PreRelease: utilfeature.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:
diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go
index b498c5158ee..17a10df2f6a 100644
--- a/pkg/kubelet/stats/cadvisor_stats_provider.go
+++ b/pkg/kubelet/stats/cadvisor_stats_provider.go
@@ -76,20 +76,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
if err != nil {
return nil, fmt.Errorf("failed to get imageFs info: %v", err)
}
-
- infos, err := p.cadvisor.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{
- IdType: cadvisorapiv2.TypeName,
- Count: 2, // 2 samples are needed to compute "instantaneous" CPU
- Recursive: true,
- })
+ infos, err := getCadvisorContainerInfo(p.cadvisor)
if err != nil {
- if _, ok := infos["/"]; ok {
- // If the failure is partial, log it and return a best-effort
- // response.
- glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err)
- } else {
- return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
- }
+ return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
@@ -352,3 +341,21 @@ func hasMemoryAndCPUInstUsage(info *cadvisorapiv2.ContainerInfo) bool {
}
return cstat.CpuInst.Usage.Total != 0 && cstat.Memory.RSS != 0
}
+
+func getCadvisorContainerInfo(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) {
+ infos, err := ca.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{
+ IdType: cadvisorapiv2.TypeName,
+ Count: 2, // 2 samples are needed to compute "instantaneous" CPU
+ Recursive: true,
+ })
+ if err != nil {
+ if _, ok := infos["/"]; ok {
+ // If the failure is partial, log it and return a best-effort
+ // response.
+ glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err)
+ } else {
+ return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
+ }
+ }
+ return infos, nil
+}
diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go
index afc364068b8..5934c2962bf 100644
--- a/pkg/kubelet/stats/cri_stats_provider.go
+++ b/pkg/kubelet/stats/cri_stats_provider.go
@@ -18,7 +18,9 @@ package stats
import (
"fmt"
+ "path"
"sort"
+ "strings"
"time"
"github.com/golang/glog"
@@ -112,6 +114,11 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
containerMap[c.Id] = c
}
+ caInfos, err := getCRICadvisorStats(p.cadvisor)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
+ }
+
for _, stats := range resp {
containerID := stats.Attributes.Id
container, found := containerMap[containerID]
@@ -132,10 +139,25 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
ps, found := sandboxIDToPodStats[podSandboxID]
if !found {
ps = buildPodStats(podSandbox)
+ // Fill stats from cadvisor is available for full set of required pod stats
+ caPodSandbox, found := caInfos[podSandboxID]
+ if !found {
+ glog.V(4).Info("Unable to find cadvisor stats for sandbox %q", podSandboxID)
+ } else {
+ p.addCadvisorPodStats(ps, &caPodSandbox)
+ }
sandboxIDToPodStats[podSandboxID] = ps
}
- containerStats := p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)
- ps.Containers = append(ps.Containers, *containerStats)
+ cs := p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)
+ // If cadvisor stats is available for the container, use it to populate
+ // container stats
+ caStats, caFound := caInfos[containerID]
+ if !caFound {
+ glog.V(4).Info("Unable to find cadvisor stats for %q", containerID)
+ } else {
+ p.addCadvisorContainerStats(cs, &caStats)
+ }
+ ps.Containers = append(ps.Containers, *cs)
}
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
@@ -201,7 +223,7 @@ func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *c
return &fsInfo
}
-// buildPodRef returns a PodStats that identifies the Pod managing cinfo
+// buildPodStats returns a PodStats that identifies the Pod managing cinfo
func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
return &statsapi.PodStats{
PodRef: statsapi.PodReference{
@@ -211,7 +233,6 @@ func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
},
// The StartTime in the summary API is the pod creation time.
StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
- // Network stats are not supported by CRI.
}
}
@@ -226,6 +247,13 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo
return s
}
+func (p *criStatsProvider) addCadvisorPodStats(
+ ps *statsapi.PodStats,
+ caPodSandbox *cadvisorapiv2.ContainerInfo,
+) {
+ ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, caPodSandbox)
+}
+
func (p *criStatsProvider) makeContainerStats(
stats *runtimeapi.ContainerStats,
container *runtimeapi.Container,
@@ -336,3 +364,44 @@ func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi
}
return result
}
+
+func (p *criStatsProvider) addCadvisorContainerStats(
+ cs *statsapi.ContainerStats,
+ caPodStats *cadvisorapiv2.ContainerInfo,
+) {
+ if caPodStats.Spec.HasCustomMetrics {
+ cs.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(caPodStats)
+ }
+
+ cpu, memory := cadvisorInfoToCPUandMemoryStats(caPodStats)
+ if cpu != nil {
+ cs.CPU = cpu
+ }
+ if memory != nil {
+ cs.Memory = memory
+ }
+}
+
+func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) {
+ stats := make(map[string]cadvisorapiv2.ContainerInfo)
+ infos, err := getCadvisorContainerInfo(ca)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
+ }
+ infos = removeTerminatedContainerInfo(infos)
+ for key, info := range infos {
+ // On systemd using devicemapper each mount into the container has an
+ // associated cgroup. We ignore them to ensure we do not get duplicate
+ // entries in our summary. For details on .mount units:
+ // http://man7.org/linux/man-pages/man5/systemd.mount.5.html
+ if strings.HasSuffix(key, ".mount") {
+ continue
+ }
+ // Build the Pod key if this container is managed by a Pod
+ if !isPodManagedContainer(&info) {
+ continue
+ }
+ stats[path.Base(key)] = info
+ }
+ return stats, nil
+}
diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go
index b966d0ba80f..5cff0ac7e90 100644
--- a/pkg/kubelet/stats/cri_stats_provider_test.go
+++ b/pkg/kubelet/stats/cri_stats_provider_test.go
@@ -22,20 +22,47 @@ import (
"time"
cadvisorfs "github.com/google/cadvisor/fs"
- "github.com/stretchr/testify/assert"
-
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+ "github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+ "k8s.io/kubernetes/pkg/kubelet/leaky"
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
)
func TestCRIListPodStats(t *testing.T) {
+ const (
+ seedRoot = 0
+ seedRuntime = 100
+ seedKubelet = 200
+ seedMisc = 300
+ seedSandbox0 = 1000
+ seedContainer0 = 2000
+ seedSandbox1 = 3000
+ seedContainer1 = 4000
+ seedContainer2 = 5000
+ seedSandbox2 = 6000
+ seedContainer3 = 7000
+ )
+
+ const (
+ pName0 = "pod0"
+ pName1 = "pod1"
+ pName2 = "pod2"
+ )
+
+ const (
+ cName0 = "container0-name"
+ cName1 = "container1-name"
+ cName2 = "container2-name"
+ cName3 = "container3-name"
+ )
+
var (
imageFsStorageUUID = "imagefs-storage-uuid"
unknownStorageUUID = "unknown-storage-uuid"
@@ -43,19 +70,19 @@ func TestCRIListPodStats(t *testing.T) {
rootFsInfo = getTestFsInfo(1000)
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
- container0 = makeFakeContainer(sandbox0, "container0-name", 0, false)
+ container0 = makeFakeContainer(sandbox0, cName0, 0, false)
containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID)
- container1 = makeFakeContainer(sandbox0, "container1-name", 0, false)
+ container1 = makeFakeContainer(sandbox0, cName1, 0, false)
containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID)
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
- container2 = makeFakeContainer(sandbox1, "container2-name", 0, false)
+ container2 = makeFakeContainer(sandbox1, cName2, 0, false)
containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID)
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
- container3 = makeFakeContainer(sandbox2, "container3-name", 0, true)
+ container3 = makeFakeContainer(sandbox2, cName3, 0, true)
containerStats3 = makeFakeContainerStats(container3, imageFsStorageUUID)
- container4 = makeFakeContainer(sandbox2, "container3-name", 1, false)
+ container4 = makeFakeContainer(sandbox2, cName3, 1, false)
containerStats4 = makeFakeContainerStats(container4, imageFsStorageUUID)
)
@@ -68,7 +95,27 @@ func TestCRIListPodStats(t *testing.T) {
fakeImageService = critest.NewFakeImageService()
)
+ infos := map[string]cadvisorapiv2.ContainerInfo{
+ "/": getTestContainerInfo(seedRoot, "", "", ""),
+ "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
+ "/system": getTestContainerInfo(seedMisc, "", "", ""),
+ sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+ container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
+ container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
+ sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+ container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
+ sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+ container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
+ }
+
+ options := cadvisorapiv2.RequestOptions{
+ IdType: cadvisorapiv2.TypeName,
+ Count: 2,
+ Recursive: true,
+ }
+
mockCadvisor.
+ On("ContainerInfoV2", "/", options).Return(infos, nil).
On("RootFsInfo").Return(rootFsInfo, nil).
On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil).
On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
@@ -117,16 +164,18 @@ func TestCRIListPodStats(t *testing.T) {
for _, s := range p0.Containers {
containerStatsMap[s.Name] = s
}
- c0 := containerStatsMap["container0-name"]
+
+ c0 := containerStatsMap[cName0]
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
- checkCRICPUAndMemoryStats(assert, c0, containerStats0)
+ checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
checkCRILogsStats(assert, c0, &rootFsInfo)
- c1 := containerStatsMap["container1-name"]
+ c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
- checkCRICPUAndMemoryStats(assert, c1, containerStats1)
+ checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c1, containerStats1, nil)
checkCRILogsStats(assert, c1, &rootFsInfo)
+ checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
@@ -134,11 +183,12 @@ func TestCRIListPodStats(t *testing.T) {
checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
c2 := p1.Containers[0]
- assert.Equal("container2-name", c2.Name)
+ assert.Equal(cName2, c2.Name)
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
- checkCRICPUAndMemoryStats(assert, c2, containerStats2)
+ checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
checkCRILogsStats(assert, c2, &rootFsInfo)
+ checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
@@ -147,12 +197,13 @@ func TestCRIListPodStats(t *testing.T) {
checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
c3 := p2.Containers[0]
- assert.Equal("container3-name", c3.Name)
+ assert.Equal(cName3, c3.Name)
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
- checkCRICPUAndMemoryStats(assert, c3, containerStats4)
+ checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
checkCRILogsStats(assert, c3, &rootFsInfo)
+ checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
mockCadvisor.AssertExpectations(t)
}
@@ -306,18 +357,16 @@ func makeFakeVolumeStats(volumeNames []string) []statsapi.VolumeStats {
return volumes
}
-func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
- assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
- assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
- assert.Zero(*actual.CPU.UsageNanoCores)
+func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *cadvisorapiv2.ContainerStats) {
+ assert.Equal(cs.Timestamp.UnixNano(), actual.CPU.Time.UnixNano())
+ assert.Equal(cs.Cpu.Usage.Total, *actual.CPU.UsageCoreNanoSeconds)
+ assert.Equal(cs.CpuInst.Usage.Total, *actual.CPU.UsageNanoCores)
- assert.Equal(cs.Memory.Timestamp, actual.Memory.Time.UnixNano())
- assert.Nil(actual.Memory.AvailableBytes)
- assert.Nil(actual.Memory.UsageBytes)
- assert.Equal(cs.Memory.WorkingSetBytes.Value, *actual.Memory.WorkingSetBytes)
- assert.Zero(*actual.Memory.RSSBytes)
- assert.Nil(actual.Memory.PageFaults)
- assert.Nil(actual.Memory.MajorPageFaults)
+ assert.Equal(cs.Memory.Usage, *actual.Memory.UsageBytes)
+ assert.Equal(cs.Memory.WorkingSet, *actual.Memory.WorkingSetBytes)
+ assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
+ assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
+ assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
}
func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
@@ -361,3 +410,10 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt
assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
}
+
+func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
+ assert.Equal(expected.Interfaces[0].RxBytes, *actual.RxBytes)
+ assert.Equal(expected.Interfaces[0].RxErrors, *actual.RxErrors)
+ assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
+ assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
+}
diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go
index 6315c315f20..e4fde7d1639 100644
--- a/pkg/kubelet/stats/helper.go
+++ b/pkg/kubelet/stats/helper.go
@@ -86,29 +86,12 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
if rootFs != nil {
// The container logs live on the node rootfs device
- result.Logs = &statsapi.FsStats{
- Time: metav1.NewTime(cstat.Timestamp),
- AvailableBytes: &rootFs.Available,
- CapacityBytes: &rootFs.Capacity,
- InodesFree: rootFs.InodesFree,
- Inodes: rootFs.Inodes,
- }
-
- if rootFs.Inodes != nil && rootFs.InodesFree != nil {
- logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
- result.Logs.InodesUsed = &logsInodesUsed
- }
+ result.Logs = buildLogsStats(cstat, rootFs)
}
if imageFs != nil {
// The container rootFs lives on the imageFs devices (which may not be the node root fs)
- result.Rootfs = &statsapi.FsStats{
- Time: metav1.NewTime(cstat.Timestamp),
- AvailableBytes: &imageFs.Available,
- CapacityBytes: &imageFs.Capacity,
- InodesFree: imageFs.InodesFree,
- Inodes: imageFs.Inodes,
- }
+ result.Rootfs = buildRootfsStats(cstat, imageFs)
}
cfs := cstat.Filesystem
@@ -274,3 +257,29 @@ func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadviso
}
return stats, nil
}
+
+func buildLogsStats(cstat *cadvisorapiv2.ContainerStats, rootFs *cadvisorapiv2.FsInfo) *statsapi.FsStats {
+ fsStats := &statsapi.FsStats{
+ Time: metav1.NewTime(cstat.Timestamp),
+ AvailableBytes: &rootFs.Available,
+ CapacityBytes: &rootFs.Capacity,
+ InodesFree: rootFs.InodesFree,
+ Inodes: rootFs.Inodes,
+ }
+
+ if rootFs.Inodes != nil && rootFs.InodesFree != nil {
+ logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
+ fsStats.InodesUsed = &logsInodesUsed
+ }
+ return fsStats
+}
+
+func buildRootfsStats(cstat *cadvisorapiv2.ContainerStats, imageFs *cadvisorapiv2.FsInfo) *statsapi.FsStats {
+ return &statsapi.FsStats{
+ Time: metav1.NewTime(cstat.Timestamp),
+ AvailableBytes: &imageFs.Available,
+ CapacityBytes: &imageFs.Capacity,
+ InodesFree: imageFs.InodesFree,
+ Inodes: imageFs.Inodes,
+ }
+}
diff --git a/pkg/kubelet/volumemanager/populator/BUILD b/pkg/kubelet/volumemanager/populator/BUILD
index f8938c1ce3d..a3bd36cc057 100644
--- a/pkg/kubelet/volumemanager/populator/BUILD
+++ b/pkg/kubelet/volumemanager/populator/BUILD
@@ -19,6 +19,7 @@ go_library(
"//pkg/kubelet/util/format:go_default_library",
"//pkg/kubelet/volumemanager/cache:go_default_library",
"//pkg/volume:go_default_library",
+ "//pkg/volume/util:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumehelper:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
diff --git a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go
index a9ecaaad69e..ac762a040fa 100644
--- a/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go
+++ b/pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go
@@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
"k8s.io/kubernetes/pkg/kubelet/volumemanager/cache"
"k8s.io/kubernetes/pkg/volume"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
@@ -419,7 +420,8 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec(
}
// getPVCExtractPV fetches the PVC object with the given namespace and name from
-// the API server extracts the name of the PV it is pointing to and returns it.
+// the API server, checks whether PVC is being deleted, extracts the name of the PV
+// it is pointing to and returns it.
// An error is returned if the PVC object's phase is not "Bound".
func (dswp *desiredStateOfWorldPopulator) getPVCExtractPV(
namespace string, claimName string) (string, types.UID, error) {
@@ -433,6 +435,23 @@ func (dswp *desiredStateOfWorldPopulator) getPVCExtractPV(
err)
}
+ if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
+ // Pods that uses a PVC that is being deleted must not be started.
+ //
+ // In case an old kubelet is running without this check or some kubelets
+ // have this feature disabled, the worst that can happen is that such
+ // pod is scheduled. This was the default behavior in 1.8 and earlier
+ // and users should not be that surprised.
+ // It should happen only in very rare case when scheduler schedules
+ // a pod and user deletes a PVC that's used by it at the same time.
+ if volumeutil.IsPVCBeingDeleted(pvc) {
+ return "", "", fmt.Errorf(
+ "can't start pod because PVC %s/%s is being deleted",
+ namespace,
+ claimName)
+ }
+ }
+
if pvc.Status.Phase != v1.ClaimBound || pvc.Spec.VolumeName == "" {
return "", "", fmt.Errorf(
diff --git a/pkg/printers/internalversion/describe.go b/pkg/printers/internalversion/describe.go
index c876dd1271c..c33b1c636ce 100644
--- a/pkg/printers/internalversion/describe.go
+++ b/pkg/printers/internalversion/describe.go
@@ -3386,6 +3386,9 @@ func describePodSecurityPolicy(psp *extensions.PodSecurityPolicy) (string, error
w.Write(LEVEL_1, "Allowed Capabilities:\t%s\n", capsToString(psp.Spec.AllowedCapabilities))
w.Write(LEVEL_1, "Allowed Volume Types:\t%s\n", fsTypeToString(psp.Spec.Volumes))
+ if len(psp.Spec.AllowedFlexVolumes) > 0 {
+ w.Write(LEVEL_1, "Allowed FlexVolume Types:\t%s\n", flexVolumesToString(psp.Spec.AllowedFlexVolumes))
+ }
w.Write(LEVEL_1, "Allow Host Network:\t%t\n", psp.Spec.HostNetwork)
w.Write(LEVEL_1, "Allow Host Ports:\t%s\n", hostPortRangeToString(psp.Spec.HostPorts))
w.Write(LEVEL_1, "Allow Host PID:\t%t\n", psp.Spec.HostPID)
@@ -3419,10 +3422,14 @@ func describePodSecurityPolicy(psp *extensions.PodSecurityPolicy) (string, error
}
func stringOrNone(s string) string {
+ return stringOrDefaultValue(s, "
")
+}
+
+func stringOrDefaultValue(s, defaultValue string) string {
if len(s) > 0 {
return s
}
- return ""
+ return defaultValue
}
func fsTypeToString(volumes []extensions.FSType) string {
@@ -3433,6 +3440,14 @@ func fsTypeToString(volumes []extensions.FSType) string {
return stringOrNone(strings.Join(strVolumes, ","))
}
+func flexVolumesToString(flexVolumes []extensions.AllowedFlexVolume) string {
+ volumes := []string{}
+ for _, flexVolume := range flexVolumes {
+ volumes = append(volumes, "driver="+flexVolume.Driver)
+ }
+ return stringOrDefaultValue(strings.Join(volumes, ","), "")
+}
+
func hostPortRangeToString(ranges []extensions.HostPortRange) string {
formattedString := ""
if ranges != nil {
diff --git a/pkg/registry/rbac/rest/storage_rbac.go b/pkg/registry/rbac/rest/storage_rbac.go
index 88fdf73d59c..b8704f8e465 100644
--- a/pkg/registry/rbac/rest/storage_rbac.go
+++ b/pkg/registry/rbac/rest/storage_rbac.go
@@ -343,6 +343,7 @@ func primeAggregatedClusterRoles(clusterRolesToAggregate map[string]string, clus
}
glog.V(1).Infof("migrating %v to %v", existingRole.Name, newName)
existingRole.Name = newName
+ existingRole.ResourceVersion = "" // clear this so the object can be created.
if _, err := clusterRoleClient.ClusterRoles().Create(existingRole); err != nil && !apierrors.IsAlreadyExists(err) {
return err
}
diff --git a/pkg/security/podsecuritypolicy/provider.go b/pkg/security/podsecuritypolicy/provider.go
index 139a453a9e2..b10aa15555c 100644
--- a/pkg/security/podsecuritypolicy/provider.go
+++ b/pkg/security/podsecuritypolicy/provider.go
@@ -233,9 +233,24 @@ func (s *simpleProvider) ValidatePodSecurityContext(pod *api.Pod, fldPath *field
fmt.Sprintf("is not allowed to be used")))
}
}
+
+ if fsType == extensions.FlexVolume && len(s.psp.Spec.AllowedFlexVolumes) > 0 {
+ found := false
+ driver := v.FlexVolume.Driver
+ for _, allowedFlexVolume := range s.psp.Spec.AllowedFlexVolumes {
+ if driver == allowedFlexVolume.Driver {
+ found = true
+ break
+ }
+ }
+ if !found {
+ allErrs = append(allErrs,
+ field.Invalid(fldPath.Child("volumes").Index(i).Child("driver"), driver,
+ "Flexvolume driver is not allowed to be used"))
+ }
+ }
}
}
-
return allErrs
}
diff --git a/pkg/security/podsecuritypolicy/provider_test.go b/pkg/security/podsecuritypolicy/provider_test.go
index b0389e33aac..6a112b67446 100644
--- a/pkg/security/podsecuritypolicy/provider_test.go
+++ b/pkg/security/podsecuritypolicy/provider_test.go
@@ -256,6 +256,18 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
failSeccompProfilePod := defaultPod()
failSeccompProfilePod.Annotations = map[string]string{api.SeccompPodAnnotationKey: "foo"}
+ podWithInvalidFlexVolumeDriver := defaultPod()
+ podWithInvalidFlexVolumeDriver.Spec.Volumes = []api.Volume{
+ {
+ Name: "flex-volume",
+ VolumeSource: api.VolumeSource{
+ FlexVolume: &api.FlexVolumeSource{
+ Driver: "example/unknown",
+ },
+ },
+ },
+ }
+
errorCases := map[string]struct {
pod *api.Pod
psp *extensions.PodSecurityPolicy
@@ -341,6 +353,16 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
psp: defaultPSP(),
expectedError: "Forbidden: seccomp may not be set",
},
+ "fail pod with disallowed flexVolume when flex volumes are allowed": {
+ pod: podWithInvalidFlexVolumeDriver,
+ psp: allowFlexVolumesPSP(false, false),
+ expectedError: "Flexvolume driver is not allowed to be used",
+ },
+ "fail pod with disallowed flexVolume when all volumes are allowed": {
+ pod: podWithInvalidFlexVolumeDriver,
+ psp: allowFlexVolumesPSP(false, true),
+ expectedError: "Flexvolume driver is not allowed to be used",
+ },
}
for k, v := range errorCases {
provider, err := NewSimpleProvider(v.psp, "namespace", NewSimpleStrategyFactory())
@@ -358,6 +380,28 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
}
}
+func allowFlexVolumesPSP(allowAllFlexVolumes, allowAllVolumes bool) *extensions.PodSecurityPolicy {
+ psp := defaultPSP()
+
+ allowedVolumes := []extensions.AllowedFlexVolume{
+ {Driver: "example/foo"},
+ {Driver: "example/bar"},
+ }
+ if allowAllFlexVolumes {
+ allowedVolumes = []extensions.AllowedFlexVolume{}
+ }
+
+ allowedVolumeType := extensions.FlexVolume
+ if allowAllVolumes {
+ allowedVolumeType = extensions.All
+ }
+
+ psp.Spec.AllowedFlexVolumes = allowedVolumes
+ psp.Spec.Volumes = []extensions.FSType{allowedVolumeType}
+
+ return psp
+}
+
func TestValidateContainerSecurityContextFailures(t *testing.T) {
// fail user strat
failUserPSP := defaultPSP()
@@ -597,6 +641,18 @@ func TestValidatePodSecurityContextSuccess(t *testing.T) {
api.SeccompPodAnnotationKey: "foo",
}
+ flexVolumePod := defaultPod()
+ flexVolumePod.Spec.Volumes = []api.Volume{
+ {
+ Name: "flex-volume",
+ VolumeSource: api.VolumeSource{
+ FlexVolume: &api.FlexVolumeSource{
+ Driver: "example/bar",
+ },
+ },
+ },
+ }
+
successCases := map[string]struct {
pod *api.Pod
psp *extensions.PodSecurityPolicy
@@ -653,6 +709,22 @@ func TestValidatePodSecurityContextSuccess(t *testing.T) {
pod: seccompPod,
psp: seccompPSP,
},
+ "flex volume driver in a whitelist (all volumes are allowed)": {
+ pod: flexVolumePod,
+ psp: allowFlexVolumesPSP(false, true),
+ },
+ "flex volume driver with empty whitelist (all volumes are allowed)": {
+ pod: flexVolumePod,
+ psp: allowFlexVolumesPSP(true, true),
+ },
+ "flex volume driver in a whitelist (only flex volumes are allowed)": {
+ pod: flexVolumePod,
+ psp: allowFlexVolumesPSP(false, false),
+ },
+ "flex volume driver with empty whitelist (only flex volumes volumes are allowed)": {
+ pod: flexVolumePod,
+ psp: allowFlexVolumesPSP(true, false),
+ },
}
for k, v := range successCases {
diff --git a/pkg/volume/aws_ebs/BUILD b/pkg/volume/aws_ebs/BUILD
index 5d45520f468..e61d65a9da2 100644
--- a/pkg/volume/aws_ebs/BUILD
+++ b/pkg/volume/aws_ebs/BUILD
@@ -46,6 +46,7 @@ go_test(
"//pkg/volume/testing:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
diff --git a/pkg/volume/aws_ebs/attacher_test.go b/pkg/volume/aws_ebs/attacher_test.go
index 16df5812578..813139e5b95 100644
--- a/pkg/volume/aws_ebs/attacher_test.go
+++ b/pkg/volume/aws_ebs/attacher_test.go
@@ -26,6 +26,7 @@ import (
volumetest "k8s.io/kubernetes/pkg/volume/testing"
"github.com/golang/glog"
+ "k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
)
@@ -340,3 +341,10 @@ func (testcase *testcase) GetVolumeLabels(volumeName aws.KubernetesVolumeID) (ma
func (testcase *testcase) GetDiskPath(volumeName aws.KubernetesVolumeID) (string, error) {
return "", errors.New("Not implemented")
}
+
+func (testcase *testcase) ResizeDisk(
+ volumeName aws.KubernetesVolumeID,
+ oldSize resource.Quantity,
+ newSize resource.Quantity) (resource.Quantity, error) {
+ return oldSize, errors.New("Not implemented")
+}
diff --git a/pkg/volume/aws_ebs/aws_ebs.go b/pkg/volume/aws_ebs/aws_ebs.go
index 5c9f2702c00..9a0091928e2 100644
--- a/pkg/volume/aws_ebs/aws_ebs.go
+++ b/pkg/volume/aws_ebs/aws_ebs.go
@@ -241,6 +241,33 @@ func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath
return volume.NewSpecFromVolume(awsVolume), nil
}
+func (plugin *awsElasticBlockStorePlugin) RequiresFSResize() bool {
+ return true
+}
+
+func (plugin *awsElasticBlockStorePlugin) ExpandVolumeDevice(
+ spec *volume.Spec,
+ newSize resource.Quantity,
+ oldSize resource.Quantity) (resource.Quantity, error) {
+ var awsVolume aws.Volumes
+
+ awsVolume, err := getCloudProvider(plugin.host.GetCloudProvider())
+
+ if err != nil {
+ return oldSize, err
+ }
+ // we don't expect to receive this call for non PVs
+ rawVolumeName := spec.PersistentVolume.Spec.AWSElasticBlockStore.VolumeID
+ volumeID := aws.KubernetesVolumeID(rawVolumeName)
+
+ if volumeID == "" {
+ return oldSize, fmt.Errorf("EBS.ExpandVolumeDevice Invalid volume id for %s", spec.Name())
+ }
+ return awsVolume.ResizeDisk(volumeID, oldSize, newSize)
+}
+
+var _ volume.ExpandableVolumePlugin = &awsElasticBlockStorePlugin{}
+
// Abstract interface to PD operations.
type ebsManager interface {
CreateVolume(provisioner *awsElasticBlockStoreProvisioner) (volumeID aws.KubernetesVolumeID, volumeSizeGB int, labels map[string]string, fstype string, err error)
diff --git a/pkg/volume/downwardapi/downwardapi_test.go b/pkg/volume/downwardapi/downwardapi_test.go
index 67a312012a2..89c73cb983b 100644
--- a/pkg/volume/downwardapi/downwardapi_test.go
+++ b/pkg/volume/downwardapi/downwardapi_test.go
@@ -63,6 +63,12 @@ func TestCanSupport(t *testing.T) {
if plugin.GetPluginName() != downwardAPIPluginName {
t.Errorf("Wrong name: %s", plugin.GetPluginName())
}
+ if !plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{DownwardAPI: &v1.DownwardAPIVolumeSource{}}}}) {
+ t.Errorf("Expected true")
+ }
+ if plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) {
+ t.Errorf("Expected false")
+ }
}
func TestDownwardAPI(t *testing.T) {
diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD
index 3c94453fd45..48a0b0ee6bf 100644
--- a/pkg/volume/util/BUILD
+++ b/pkg/volume/util/BUILD
@@ -14,6 +14,7 @@ go_library(
"device_util_unsupported.go",
"doc.go",
"error.go",
+ "finalizer.go",
"fs_unsupported.go",
"io_util.go",
"metrics.go",
@@ -61,6 +62,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
+ "finalizer_test.go",
"util_test.go",
] + select({
"@io_bazel_rules_go//go/platform:linux_amd64": [
@@ -74,6 +76,7 @@ go_test(
deps = [
"//pkg/apis/core/install:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
+ "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
diff --git a/pkg/volume/util/finalizer.go b/pkg/volume/util/finalizer.go
new file mode 100644
index 00000000000..84631545060
--- /dev/null
+++ b/pkg/volume/util/finalizer.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+ "k8s.io/api/core/v1"
+)
+
+const (
+ // Name of finalizer on PVCs that have a running pod.
+ PVCProtectionFinalizer = "kubernetes.io/pvc-protection"
+)
+
+// IsPVCBeingDeleted returns:
+// true: in case PVC is being deleted, i.e. ObjectMeta.DeletionTimestamp is set
+// false: in case PVC is not being deleted, i.e. ObjectMeta.DeletionTimestamp is nil
+func IsPVCBeingDeleted(pvc *v1.PersistentVolumeClaim) bool {
+ return pvc.ObjectMeta.DeletionTimestamp != nil
+}
+
+// IsProtectionFinalizerPresent returns true in case PVCProtectionFinalizer is
+// present among the pvc.Finalizers
+func IsProtectionFinalizerPresent(pvc *v1.PersistentVolumeClaim) bool {
+ for _, finalizer := range pvc.Finalizers {
+ if finalizer == PVCProtectionFinalizer {
+ return true
+ }
+ }
+ return false
+}
+
+// RemoveProtectionFinalizer returns pvc without PVCProtectionFinalizer in case
+// it's present in pvc.Finalizers. It expects that pvc is writable (i.e. is not
+// informer's cached copy.)
+func RemoveProtectionFinalizer(pvc *v1.PersistentVolumeClaim) {
+ newFinalizers := make([]string, 0)
+ for _, finalizer := range pvc.Finalizers {
+ if finalizer != PVCProtectionFinalizer {
+ newFinalizers = append(newFinalizers, finalizer)
+ }
+ }
+ if len(newFinalizers) == 0 {
+ // Sanitize for unit tests so we don't need to distinguish empty array
+ // and nil.
+ newFinalizers = nil
+ }
+ pvc.Finalizers = newFinalizers
+}
+
+// AddProtectionFinalizer adds PVCProtectionFinalizer to pvc. It expects that
+// pvc is writable (i.e. is not informer's cached copy.)
+func AddProtectionFinalizer(pvc *v1.PersistentVolumeClaim) {
+ pvc.Finalizers = append(pvc.Finalizers, PVCProtectionFinalizer)
+}
diff --git a/pkg/volume/util/finalizer_test.go b/pkg/volume/util/finalizer_test.go
new file mode 100644
index 00000000000..210ea3b3e63
--- /dev/null
+++ b/pkg/volume/util/finalizer_test.go
@@ -0,0 +1,231 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/davecgh/go-spew/spew"
+ "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+ arbitraryTime = metav1.Date(2017, 11, 1, 14, 28, 47, 0, time.FixedZone("CET", 0))
+)
+
+func TestIsPVCBeingDeleted(t *testing.T) {
+ tests := []struct {
+ pvc *v1.PersistentVolumeClaim
+ want bool
+ }{
+ {
+ pvc: &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ DeletionTimestamp: nil,
+ },
+ },
+ want: false,
+ },
+ {
+ pvc: &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ DeletionTimestamp: &arbitraryTime,
+ },
+ },
+ want: true,
+ },
+ }
+ for _, tt := range tests {
+ if got := IsPVCBeingDeleted(tt.pvc); got != tt.want {
+ t.Errorf("IsPVCBeingDeleted(%v) = %v WANT %v", tt.pvc, got, tt.want)
+ }
+ }
+}
+
+func TestAddProtectionFinalizer(t *testing.T) {
+ tests := []struct {
+ name string
+ pvc *v1.PersistentVolumeClaim
+ want *v1.PersistentVolumeClaim
+ }{
+ {
+ "PVC without finalizer",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ },
+ },
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{PVCProtectionFinalizer},
+ },
+ },
+ },
+ {
+ "PVC with some finalizers",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
+ },
+ },
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ got := test.pvc.DeepCopy()
+ AddProtectionFinalizer(got)
+ if !reflect.DeepEqual(got, test.want) {
+ t.Errorf("Test %q: expected:\n%s\n\ngot:\n%s", test.name, spew.Sdump(test.want), spew.Sdump(got))
+ }
+ }
+}
+
+func TestRemoveProtectionFinalizer(t *testing.T) {
+ tests := []struct {
+ name string
+ pvc *v1.PersistentVolumeClaim
+ want *v1.PersistentVolumeClaim
+ }{
+ {
+ "PVC without finalizer",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ },
+ },
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ },
+ },
+ },
+ {
+ "PVC with finalizer",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{PVCProtectionFinalizer},
+ },
+ },
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ },
+ },
+ },
+ {
+ "PVC with many finalizers",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
+ },
+ },
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
+ },
+ },
+ },
+ }
+ for _, test := range tests {
+ got := test.pvc.DeepCopy()
+ RemoveProtectionFinalizer(got)
+ if !reflect.DeepEqual(got, test.want) {
+ t.Errorf("Test %q: expected:\n%s\n\ngot:\n%s", test.name, spew.Sdump(test.want), spew.Sdump(got))
+ }
+ }
+}
+
+func TestIsProtectionFinalizerPresent(t *testing.T) {
+ tests := []struct {
+ name string
+ pvc *v1.PersistentVolumeClaim
+ want bool
+ }{
+ {
+ "PVC without finalizer",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ },
+ },
+ false,
+ },
+ {
+ "PVC with many unrelated finalizers",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
+ },
+ },
+ false,
+ },
+ {
+ "PVC with many finalizers",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
+ },
+ },
+ true,
+ },
+ {
+ "PVC with finalizer",
+ &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "pvc",
+ Namespace: "ns",
+ Finalizers: []string{PVCProtectionFinalizer},
+ },
+ },
+ true,
+ },
+ }
+ for _, test := range tests {
+ got := IsProtectionFinalizerPresent(test.pvc)
+ if got != test.want {
+ t.Errorf("Test %q: expected %v, got %v", test.name, test.want, got)
+ }
+ }
+}
diff --git a/plugin/BUILD b/plugin/BUILD
index 764a8e3283f..03b74856108 100644
--- a/plugin/BUILD
+++ b/plugin/BUILD
@@ -29,6 +29,7 @@ filegroup(
"//plugin/pkg/admission/noderestriction:all-srcs",
"//plugin/pkg/admission/persistentvolume/label:all-srcs",
"//plugin/pkg/admission/persistentvolume/resize:all-srcs",
+ "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:all-srcs",
"//plugin/pkg/admission/podnodeselector:all-srcs",
"//plugin/pkg/admission/podpreset:all-srcs",
"//plugin/pkg/admission/podtolerationrestriction:all-srcs",
diff --git a/plugin/pkg/admission/persistentvolume/label/BUILD b/plugin/pkg/admission/persistentvolume/label/BUILD
index 0c04cbc99d2..b88f19f863e 100644
--- a/plugin/pkg/admission/persistentvolume/label/BUILD
+++ b/plugin/pkg/admission/persistentvolume/label/BUILD
@@ -34,6 +34,7 @@ go_test(
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/cloudprovider/providers/aws:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
diff --git a/plugin/pkg/admission/persistentvolume/label/admission_test.go b/plugin/pkg/admission/persistentvolume/label/admission_test.go
index d13dcc30384..e349c756a4d 100644
--- a/plugin/pkg/admission/persistentvolume/label/admission_test.go
+++ b/plugin/pkg/admission/persistentvolume/label/admission_test.go
@@ -21,6 +21,7 @@ import (
"fmt"
+ "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/admission"
@@ -67,6 +68,13 @@ func (c *mockVolumes) DisksAreAttached(nodeDisks map[types.NodeName][]aws.Kubern
return nil, fmt.Errorf("not implemented")
}
+func (c *mockVolumes) ResizeDisk(
+ diskName aws.KubernetesVolumeID,
+ oldSize resource.Quantity,
+ newSize resource.Quantity) (resource.Quantity, error) {
+ return oldSize, nil
+}
+
func mockVolumeFailure(err error) *mockVolumes {
return &mockVolumes{volumeLabelsError: err}
}
diff --git a/plugin/pkg/admission/persistentvolume/resize/admission.go b/plugin/pkg/admission/persistentvolume/resize/admission.go
index 7227f96696a..9909f5f3fb7 100644
--- a/plugin/pkg/admission/persistentvolume/resize/admission.go
+++ b/plugin/pkg/admission/persistentvolume/resize/admission.go
@@ -156,6 +156,10 @@ func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVol
if pv.Spec.GCEPersistentDisk != nil {
return true
}
- return false
+ if pv.Spec.AWSElasticBlockStore != nil {
+ return true
+ }
+
+ return false
}
diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD
new file mode 100644
index 00000000000..e13f63e5934
--- /dev/null
+++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD
@@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["admission.go"],
+ importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//pkg/apis/core:go_default_library",
+ "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+ "//pkg/client/listers/core/internalversion:go_default_library",
+ "//pkg/features:go_default_library",
+ "//pkg/kubeapiserver/admission:go_default_library",
+ "//pkg/volume/util:go_default_library",
+ "//vendor/github.com/golang/glog:go_default_library",
+ "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+ "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+ ],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["admission_test.go"],
+ importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection",
+ library = ":go_default_library",
+ deps = [
+ "//pkg/apis/core:go_default_library",
+ "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+ "//pkg/controller:go_default_library",
+ "//pkg/volume/util:go_default_library",
+ "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+ "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+ "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+ ],
+)
+
+filegroup(
+ name = "package-srcs",
+ srcs = glob(["**"]),
+ tags = ["automanaged"],
+ visibility = ["//visibility:private"],
+)
+
+filegroup(
+ name = "all-srcs",
+ srcs = [":package-srcs"],
+ tags = ["automanaged"],
+ visibility = ["//visibility:public"],
+)
diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go
new file mode 100644
index 00000000000..218bca4b829
--- /dev/null
+++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go
@@ -0,0 +1,111 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvcprotection
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/golang/glog"
+
+ admission "k8s.io/apiserver/pkg/admission"
+ "k8s.io/apiserver/pkg/util/feature"
+ api "k8s.io/kubernetes/pkg/apis/core"
+ informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+ corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion"
+ "k8s.io/kubernetes/pkg/features"
+ kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
+)
+
+const (
+ // PluginName is the name of this admission controller plugin
+ PluginName = "PVCProtection"
+)
+
+// Register registers a plugin
+func Register(plugins *admission.Plugins) {
+ plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
+ plugin := newPlugin()
+ return plugin, nil
+ })
+}
+
+// pvcProtectionPlugin holds state for and implements the admission plugin.
+type pvcProtectionPlugin struct {
+ *admission.Handler
+ lister corelisters.PersistentVolumeClaimLister
+}
+
+var _ admission.Interface = &pvcProtectionPlugin{}
+var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&pvcProtectionPlugin{})
+
+// newPlugin creates a new admission plugin.
+func newPlugin() *pvcProtectionPlugin {
+ return &pvcProtectionPlugin{
+ Handler: admission.NewHandler(admission.Create),
+ }
+}
+
+func (c *pvcProtectionPlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) {
+ informer := f.Core().InternalVersion().PersistentVolumeClaims()
+ c.lister = informer.Lister()
+ c.SetReadyFunc(informer.Informer().HasSynced)
+}
+
+// ValidateInitialization ensures lister is set.
+func (c *pvcProtectionPlugin) ValidateInitialization() error {
+ if c.lister == nil {
+ return fmt.Errorf("missing lister")
+ }
+ return nil
+}
+
+// Admit sets finalizer on all PVCs. The finalizer is removed by
+// PVCProtectionController when it's not referenced by any pod.
+//
+// This prevents users from deleting a PVC that's used by a running pod.
+func (c *pvcProtectionPlugin) Admit(a admission.Attributes) error {
+ if !feature.DefaultFeatureGate.Enabled(features.PVCProtection) {
+ return nil
+ }
+
+ if a.GetResource().GroupResource() != api.Resource("persistentvolumeclaims") {
+ return nil
+ }
+
+ if len(a.GetSubresource()) != 0 {
+ return nil
+ }
+
+ pvc, ok := a.GetObject().(*api.PersistentVolumeClaim)
+ // if we can't convert then we don't handle this object so just return
+ if !ok {
+ return nil
+ }
+
+ for _, f := range pvc.Finalizers {
+ if f == volumeutil.PVCProtectionFinalizer {
+ // Finalizer is already present, nothing to do
+ return nil
+ }
+ }
+
+ glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name)
+ pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer)
+ return nil
+}
diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go
new file mode 100644
index 00000000000..0815c40615b
--- /dev/null
+++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go
@@ -0,0 +1,106 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvcprotection
+
+import (
+ "fmt"
+ "reflect"
+ "testing"
+
+ "github.com/davecgh/go-spew/spew"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apiserver/pkg/admission"
+ "k8s.io/apiserver/pkg/util/feature"
+ api "k8s.io/kubernetes/pkg/apis/core"
+ informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+ "k8s.io/kubernetes/pkg/controller"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
+)
+
+func TestAdmit(t *testing.T) {
+ claim := &api.PersistentVolumeClaim{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "PersistentVolumeClaim",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "claim",
+ Namespace: "ns",
+ },
+ }
+ claimWithFinalizer := claim.DeepCopy()
+ claimWithFinalizer.Finalizers = []string{volumeutil.PVCProtectionFinalizer}
+
+ tests := []struct {
+ name string
+ object runtime.Object
+ expectedObject runtime.Object
+ featureEnabled bool
+ }{
+ {
+ "create -> add finalizer",
+ claim,
+ claimWithFinalizer,
+ true,
+ },
+ {
+ "finalizer already exists -> no new finalizer",
+ claimWithFinalizer,
+ claimWithFinalizer,
+ true,
+ },
+ {
+ "disabled feature -> no finalizer",
+ claim,
+ claim,
+ false,
+ },
+ }
+
+ ctrl := newPlugin()
+ informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())
+ ctrl.SetInternalKubeInformerFactory(informerFactory)
+
+ for _, test := range tests {
+ feature.DefaultFeatureGate.Set(fmt.Sprintf("PVCProtection=%v", test.featureEnabled))
+ obj := test.object.DeepCopyObject()
+ attrs := admission.NewAttributesRecord(
+ obj, // new object
+ obj.DeepCopyObject(), // old object, copy to be sure it's not modified
+ api.Kind("PersistentVolumeClaim").WithVersion("version"),
+ claim.Namespace,
+ claim.Name,
+ api.Resource("persistentvolumeclaims").WithVersion("version"),
+ "", // subresource
+ admission.Create,
+ nil, // userInfo
+ )
+
+ err := ctrl.Admit(attrs)
+ if err != nil {
+ t.Errorf("Test %q: got unexpected error: %v", test.name, err)
+ }
+ if !reflect.DeepEqual(test.expectedObject, obj) {
+ t.Errorf("Test %q: Expected object:\n%s\ngot:\n%s", test.name, spew.Sdump(test.expectedObject), spew.Sdump(obj))
+ }
+ }
+
+ // Disable the feature for rest of the tests.
+ // TODO: remove after alpha
+ feature.DefaultFeatureGate.Set("PVCProtection=false")
+}
diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go
index f1270f611b7..cc0813f5de1 100644
--- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go
+++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go
@@ -315,6 +315,16 @@ func buildControllerRoles() ([]rbac.ClusterRole, []rbac.ClusterRoleBinding) {
eventsRule(),
},
})
+ if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
+ addControllerRole(&controllerRoles, &controllerRoleBindings, rbac.ClusterRole{
+ ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "pvc-protection-controller"},
+ Rules: []rbac.PolicyRule{
+ rbac.NewRule("get", "list", "watch", "update").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(),
+ rbac.NewRule("list", "watch", "get").Groups(legacyGroup).Resources("pods").RuleOrDie(),
+ eventsRule(),
+ },
+ })
+ }
return controllerRoles, controllerRoleBindings
}
diff --git a/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go b/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go
index c61147fcfb0..ba5998e2773 100644
--- a/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go
+++ b/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go
@@ -25,6 +25,7 @@ limitations under the License.
k8s.io/kubernetes/vendor/k8s.io/api/extensions/v1beta1/generated.proto
It has these top-level messages:
+ AllowedFlexVolume
AllowedHostPath
CustomMetricCurrentStatus
CustomMetricCurrentStatusList
@@ -113,253 +114,258 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+func (m *AllowedFlexVolume) Reset() { *m = AllowedFlexVolume{} }
+func (*AllowedFlexVolume) ProtoMessage() {}
+func (*AllowedFlexVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} }
+
func (m *AllowedHostPath) Reset() { *m = AllowedHostPath{} }
func (*AllowedHostPath) ProtoMessage() {}
-func (*AllowedHostPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} }
+func (*AllowedHostPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{1} }
func (m *CustomMetricCurrentStatus) Reset() { *m = CustomMetricCurrentStatus{} }
func (*CustomMetricCurrentStatus) ProtoMessage() {}
func (*CustomMetricCurrentStatus) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{1}
+ return fileDescriptorGenerated, []int{2}
}
func (m *CustomMetricCurrentStatusList) Reset() { *m = CustomMetricCurrentStatusList{} }
func (*CustomMetricCurrentStatusList) ProtoMessage() {}
func (*CustomMetricCurrentStatusList) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{2}
+ return fileDescriptorGenerated, []int{3}
}
func (m *CustomMetricTarget) Reset() { *m = CustomMetricTarget{} }
func (*CustomMetricTarget) ProtoMessage() {}
-func (*CustomMetricTarget) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} }
+func (*CustomMetricTarget) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} }
func (m *CustomMetricTargetList) Reset() { *m = CustomMetricTargetList{} }
func (*CustomMetricTargetList) ProtoMessage() {}
-func (*CustomMetricTargetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} }
+func (*CustomMetricTargetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} }
func (m *DaemonSet) Reset() { *m = DaemonSet{} }
func (*DaemonSet) ProtoMessage() {}
-func (*DaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} }
+func (*DaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} }
func (m *DaemonSetCondition) Reset() { *m = DaemonSetCondition{} }
func (*DaemonSetCondition) ProtoMessage() {}
-func (*DaemonSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} }
+func (*DaemonSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} }
func (m *DaemonSetList) Reset() { *m = DaemonSetList{} }
func (*DaemonSetList) ProtoMessage() {}
-func (*DaemonSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} }
+func (*DaemonSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} }
func (m *DaemonSetSpec) Reset() { *m = DaemonSetSpec{} }
func (*DaemonSetSpec) ProtoMessage() {}
-func (*DaemonSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} }
+func (*DaemonSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{9} }
func (m *DaemonSetStatus) Reset() { *m = DaemonSetStatus{} }
func (*DaemonSetStatus) ProtoMessage() {}
-func (*DaemonSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{9} }
+func (*DaemonSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{10} }
func (m *DaemonSetUpdateStrategy) Reset() { *m = DaemonSetUpdateStrategy{} }
func (*DaemonSetUpdateStrategy) ProtoMessage() {}
func (*DaemonSetUpdateStrategy) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{10}
+ return fileDescriptorGenerated, []int{11}
}
func (m *Deployment) Reset() { *m = Deployment{} }
func (*Deployment) ProtoMessage() {}
-func (*Deployment) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} }
+func (*Deployment) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
func (m *DeploymentCondition) Reset() { *m = DeploymentCondition{} }
func (*DeploymentCondition) ProtoMessage() {}
-func (*DeploymentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
+func (*DeploymentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
func (m *DeploymentList) Reset() { *m = DeploymentList{} }
func (*DeploymentList) ProtoMessage() {}
-func (*DeploymentList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
+func (*DeploymentList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
func (m *DeploymentRollback) Reset() { *m = DeploymentRollback{} }
func (*DeploymentRollback) ProtoMessage() {}
-func (*DeploymentRollback) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
+func (*DeploymentRollback) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} }
func (m *DeploymentSpec) Reset() { *m = DeploymentSpec{} }
func (*DeploymentSpec) ProtoMessage() {}
-func (*DeploymentSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} }
+func (*DeploymentSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} }
func (m *DeploymentStatus) Reset() { *m = DeploymentStatus{} }
func (*DeploymentStatus) ProtoMessage() {}
-func (*DeploymentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} }
+func (*DeploymentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} }
func (m *DeploymentStrategy) Reset() { *m = DeploymentStrategy{} }
func (*DeploymentStrategy) ProtoMessage() {}
-func (*DeploymentStrategy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} }
+func (*DeploymentStrategy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} }
func (m *FSGroupStrategyOptions) Reset() { *m = FSGroupStrategyOptions{} }
func (*FSGroupStrategyOptions) ProtoMessage() {}
-func (*FSGroupStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} }
+func (*FSGroupStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} }
func (m *HTTPIngressPath) Reset() { *m = HTTPIngressPath{} }
func (*HTTPIngressPath) ProtoMessage() {}
-func (*HTTPIngressPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} }
+func (*HTTPIngressPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} }
func (m *HTTPIngressRuleValue) Reset() { *m = HTTPIngressRuleValue{} }
func (*HTTPIngressRuleValue) ProtoMessage() {}
-func (*HTTPIngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} }
+func (*HTTPIngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} }
func (m *HostPortRange) Reset() { *m = HostPortRange{} }
func (*HostPortRange) ProtoMessage() {}
-func (*HostPortRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} }
+func (*HostPortRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} }
func (m *IDRange) Reset() { *m = IDRange{} }
func (*IDRange) ProtoMessage() {}
-func (*IDRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} }
+func (*IDRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} }
func (m *IPBlock) Reset() { *m = IPBlock{} }
func (*IPBlock) ProtoMessage() {}
-func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} }
+func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} }
func (m *Ingress) Reset() { *m = Ingress{} }
func (*Ingress) ProtoMessage() {}
-func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} }
+func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} }
func (m *IngressBackend) Reset() { *m = IngressBackend{} }
func (*IngressBackend) ProtoMessage() {}
-func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} }
+func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} }
func (m *IngressList) Reset() { *m = IngressList{} }
func (*IngressList) ProtoMessage() {}
-func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} }
+func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} }
func (m *IngressRule) Reset() { *m = IngressRule{} }
func (*IngressRule) ProtoMessage() {}
-func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} }
+func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} }
func (m *IngressRuleValue) Reset() { *m = IngressRuleValue{} }
func (*IngressRuleValue) ProtoMessage() {}
-func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} }
+func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} }
func (m *IngressSpec) Reset() { *m = IngressSpec{} }
func (*IngressSpec) ProtoMessage() {}
-func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} }
+func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} }
func (m *IngressStatus) Reset() { *m = IngressStatus{} }
func (*IngressStatus) ProtoMessage() {}
-func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} }
+func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} }
func (m *IngressTLS) Reset() { *m = IngressTLS{} }
func (*IngressTLS) ProtoMessage() {}
-func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} }
+func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} }
func (m *NetworkPolicy) Reset() { *m = NetworkPolicy{} }
func (*NetworkPolicy) ProtoMessage() {}
-func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} }
+func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} }
func (m *NetworkPolicyEgressRule) Reset() { *m = NetworkPolicyEgressRule{} }
func (*NetworkPolicyEgressRule) ProtoMessage() {}
func (*NetworkPolicyEgressRule) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{33}
+ return fileDescriptorGenerated, []int{34}
}
func (m *NetworkPolicyIngressRule) Reset() { *m = NetworkPolicyIngressRule{} }
func (*NetworkPolicyIngressRule) ProtoMessage() {}
func (*NetworkPolicyIngressRule) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{34}
+ return fileDescriptorGenerated, []int{35}
}
func (m *NetworkPolicyList) Reset() { *m = NetworkPolicyList{} }
func (*NetworkPolicyList) ProtoMessage() {}
-func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} }
+func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} }
func (m *NetworkPolicyPeer) Reset() { *m = NetworkPolicyPeer{} }
func (*NetworkPolicyPeer) ProtoMessage() {}
-func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} }
+func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} }
func (m *NetworkPolicyPort) Reset() { *m = NetworkPolicyPort{} }
func (*NetworkPolicyPort) ProtoMessage() {}
-func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} }
+func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} }
func (m *NetworkPolicySpec) Reset() { *m = NetworkPolicySpec{} }
func (*NetworkPolicySpec) ProtoMessage() {}
-func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} }
+func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} }
func (m *PodSecurityPolicy) Reset() { *m = PodSecurityPolicy{} }
func (*PodSecurityPolicy) ProtoMessage() {}
-func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} }
+func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} }
func (m *PodSecurityPolicyList) Reset() { *m = PodSecurityPolicyList{} }
func (*PodSecurityPolicyList) ProtoMessage() {}
-func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} }
+func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} }
func (m *PodSecurityPolicySpec) Reset() { *m = PodSecurityPolicySpec{} }
func (*PodSecurityPolicySpec) ProtoMessage() {}
-func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} }
+func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} }
func (m *ReplicaSet) Reset() { *m = ReplicaSet{} }
func (*ReplicaSet) ProtoMessage() {}
-func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} }
+func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} }
func (m *ReplicaSetCondition) Reset() { *m = ReplicaSetCondition{} }
func (*ReplicaSetCondition) ProtoMessage() {}
-func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} }
+func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} }
func (m *ReplicaSetList) Reset() { *m = ReplicaSetList{} }
func (*ReplicaSetList) ProtoMessage() {}
-func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} }
+func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} }
func (m *ReplicaSetSpec) Reset() { *m = ReplicaSetSpec{} }
func (*ReplicaSetSpec) ProtoMessage() {}
-func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} }
+func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} }
func (m *ReplicaSetStatus) Reset() { *m = ReplicaSetStatus{} }
func (*ReplicaSetStatus) ProtoMessage() {}
-func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} }
+func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} }
func (m *ReplicationControllerDummy) Reset() { *m = ReplicationControllerDummy{} }
func (*ReplicationControllerDummy) ProtoMessage() {}
func (*ReplicationControllerDummy) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{47}
+ return fileDescriptorGenerated, []int{48}
}
func (m *RollbackConfig) Reset() { *m = RollbackConfig{} }
func (*RollbackConfig) ProtoMessage() {}
-func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{48} }
+func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} }
func (m *RollingUpdateDaemonSet) Reset() { *m = RollingUpdateDaemonSet{} }
func (*RollingUpdateDaemonSet) ProtoMessage() {}
-func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} }
+func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} }
func (m *RollingUpdateDeployment) Reset() { *m = RollingUpdateDeployment{} }
func (*RollingUpdateDeployment) ProtoMessage() {}
func (*RollingUpdateDeployment) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{50}
+ return fileDescriptorGenerated, []int{51}
}
func (m *RunAsUserStrategyOptions) Reset() { *m = RunAsUserStrategyOptions{} }
func (*RunAsUserStrategyOptions) ProtoMessage() {}
func (*RunAsUserStrategyOptions) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{51}
+ return fileDescriptorGenerated, []int{52}
}
func (m *SELinuxStrategyOptions) Reset() { *m = SELinuxStrategyOptions{} }
func (*SELinuxStrategyOptions) ProtoMessage() {}
-func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} }
+func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} }
func (m *Scale) Reset() { *m = Scale{} }
func (*Scale) ProtoMessage() {}
-func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} }
+func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} }
func (m *ScaleSpec) Reset() { *m = ScaleSpec{} }
func (*ScaleSpec) ProtoMessage() {}
-func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} }
+func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} }
func (m *ScaleStatus) Reset() { *m = ScaleStatus{} }
func (*ScaleStatus) ProtoMessage() {}
-func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} }
+func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} }
func (m *SupplementalGroupsStrategyOptions) Reset() { *m = SupplementalGroupsStrategyOptions{} }
func (*SupplementalGroupsStrategyOptions) ProtoMessage() {}
func (*SupplementalGroupsStrategyOptions) Descriptor() ([]byte, []int) {
- return fileDescriptorGenerated, []int{56}
+ return fileDescriptorGenerated, []int{57}
}
func init() {
+ proto.RegisterType((*AllowedFlexVolume)(nil), "k8s.io.api.extensions.v1beta1.AllowedFlexVolume")
proto.RegisterType((*AllowedHostPath)(nil), "k8s.io.api.extensions.v1beta1.AllowedHostPath")
proto.RegisterType((*CustomMetricCurrentStatus)(nil), "k8s.io.api.extensions.v1beta1.CustomMetricCurrentStatus")
proto.RegisterType((*CustomMetricCurrentStatusList)(nil), "k8s.io.api.extensions.v1beta1.CustomMetricCurrentStatusList")
@@ -418,6 +424,28 @@ func init() {
proto.RegisterType((*ScaleStatus)(nil), "k8s.io.api.extensions.v1beta1.ScaleStatus")
proto.RegisterType((*SupplementalGroupsStrategyOptions)(nil), "k8s.io.api.extensions.v1beta1.SupplementalGroupsStrategyOptions")
}
+func (m *AllowedFlexVolume) Marshal() (dAtA []byte, err error) {
+ size := m.Size()
+ dAtA = make([]byte, size)
+ n, err := m.MarshalTo(dAtA)
+ if err != nil {
+ return nil, err
+ }
+ return dAtA[:n], nil
+}
+
+func (m *AllowedFlexVolume) MarshalTo(dAtA []byte) (int, error) {
+ var i int
+ _ = i
+ var l int
+ _ = l
+ dAtA[i] = 0xa
+ i++
+ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Driver)))
+ i += copy(dAtA[i:], m.Driver)
+ return i, nil
+}
+
func (m *AllowedHostPath) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@@ -2207,6 +2235,20 @@ func (m *PodSecurityPolicySpec) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
+ if len(m.AllowedFlexVolumes) > 0 {
+ for _, msg := range m.AllowedFlexVolumes {
+ dAtA[i] = 0x92
+ i++
+ dAtA[i] = 0x1
+ i++
+ i = encodeVarintGenerated(dAtA, i, uint64(msg.Size()))
+ n, err := msg.MarshalTo(dAtA[i:])
+ if err != nil {
+ return 0, err
+ }
+ i += n
+ }
+ }
return i, nil
}
@@ -2763,6 +2805,14 @@ func encodeVarintGenerated(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
return offset + 1
}
+func (m *AllowedFlexVolume) Size() (n int) {
+ var l int
+ _ = l
+ l = len(m.Driver)
+ n += 1 + l + sovGenerated(uint64(l))
+ return n
+}
+
func (m *AllowedHostPath) Size() (n int) {
var l int
_ = l
@@ -3401,6 +3451,12 @@ func (m *PodSecurityPolicySpec) Size() (n int) {
n += 2 + l + sovGenerated(uint64(l))
}
}
+ if len(m.AllowedFlexVolumes) > 0 {
+ for _, e := range m.AllowedFlexVolumes {
+ l = e.Size()
+ n += 2 + l + sovGenerated(uint64(l))
+ }
+ }
return n
}
@@ -3605,6 +3661,16 @@ func sovGenerated(x uint64) (n int) {
func sozGenerated(x uint64) (n int) {
return sovGenerated(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
+func (this *AllowedFlexVolume) String() string {
+ if this == nil {
+ return "nil"
+ }
+ s := strings.Join([]string{`&AllowedFlexVolume{`,
+ `Driver:` + fmt.Sprintf("%v", this.Driver) + `,`,
+ `}`,
+ }, "")
+ return s
+}
func (this *AllowedHostPath) String() string {
if this == nil {
return "nil"
@@ -4122,6 +4188,7 @@ func (this *PodSecurityPolicySpec) String() string {
`DefaultAllowPrivilegeEscalation:` + valueToStringGenerated(this.DefaultAllowPrivilegeEscalation) + `,`,
`AllowPrivilegeEscalation:` + valueToStringGenerated(this.AllowPrivilegeEscalation) + `,`,
`AllowedHostPaths:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AllowedHostPaths), "AllowedHostPath", "AllowedHostPath", 1), `&`, ``, 1) + `,`,
+ `AllowedFlexVolumes:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AllowedFlexVolumes), "AllowedFlexVolume", "AllowedFlexVolume", 1), `&`, ``, 1) + `,`,
`}`,
}, "")
return s
@@ -4316,6 +4383,85 @@ func valueToStringGenerated(v interface{}) string {
pv := reflect.Indirect(rv).Interface()
return fmt.Sprintf("*%v", pv)
}
+func (m *AllowedFlexVolume) Unmarshal(dAtA []byte) error {
+ l := len(dAtA)
+ iNdEx := 0
+ for iNdEx < l {
+ preIndex := iNdEx
+ var wire uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowGenerated
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ wire |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ fieldNum := int32(wire >> 3)
+ wireType := int(wire & 0x7)
+ if wireType == 4 {
+ return fmt.Errorf("proto: AllowedFlexVolume: wiretype end group for non-group")
+ }
+ if fieldNum <= 0 {
+ return fmt.Errorf("proto: AllowedFlexVolume: illegal tag %d (wire type %d)", fieldNum, wire)
+ }
+ switch fieldNum {
+ case 1:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field Driver", wireType)
+ }
+ var stringLen uint64
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowGenerated
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ stringLen |= (uint64(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ intStringLen := int(stringLen)
+ if intStringLen < 0 {
+ return ErrInvalidLengthGenerated
+ }
+ postIndex := iNdEx + intStringLen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.Driver = string(dAtA[iNdEx:postIndex])
+ iNdEx = postIndex
+ default:
+ iNdEx = preIndex
+ skippy, err := skipGenerated(dAtA[iNdEx:])
+ if err != nil {
+ return err
+ }
+ if skippy < 0 {
+ return ErrInvalidLengthGenerated
+ }
+ if (iNdEx + skippy) > l {
+ return io.ErrUnexpectedEOF
+ }
+ iNdEx += skippy
+ }
+ }
+
+ if iNdEx > l {
+ return io.ErrUnexpectedEOF
+ }
+ return nil
+}
func (m *AllowedHostPath) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@@ -10129,6 +10275,37 @@ func (m *PodSecurityPolicySpec) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
+ case 18:
+ if wireType != 2 {
+ return fmt.Errorf("proto: wrong wireType = %d for field AllowedFlexVolumes", wireType)
+ }
+ var msglen int
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowGenerated
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ msglen |= (int(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
+ if msglen < 0 {
+ return ErrInvalidLengthGenerated
+ }
+ postIndex := iNdEx + msglen
+ if postIndex > l {
+ return io.ErrUnexpectedEOF
+ }
+ m.AllowedFlexVolumes = append(m.AllowedFlexVolumes, AllowedFlexVolume{})
+ if err := m.AllowedFlexVolumes[len(m.AllowedFlexVolumes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+ return err
+ }
+ iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
@@ -12108,226 +12285,229 @@ func init() {
}
var fileDescriptorGenerated = []byte{
- // 3523 bytes of a gzipped FileDescriptorProto
+ // 3571 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0xcd, 0x6f, 0x1c, 0x47,
0x76, 0x57, 0xcf, 0x0c, 0x39, 0xc3, 0x47, 0xf1, 0xab, 0x28, 0x93, 0x63, 0xca, 0xe2, 0xc8, 0x6d,
0x40, 0x91, 0x1c, 0x69, 0xc6, 0x92, 0x2d, 0x59, 0xb1, 0x10, 0x3b, 0x1c, 0x52, 0x1f, 0x74, 0xf8,
0xa5, 0x1a, 0x52, 0x71, 0x8c, 0xc8, 0x71, 0x73, 0xa6, 0x38, 0x6c, 0xb1, 0xa7, 0xbb, 0xdd, 0x5d,
- 0x4d, 0x73, 0x2e, 0x41, 0x0e, 0x41, 0x80, 0x00, 0x09, 0x92, 0x1c, 0x9c, 0x38, 0xb7, 0xf8, 0x92,
- 0x53, 0x82, 0xe4, 0x96, 0x1c, 0x0c, 0x03, 0x01, 0x1c, 0x40, 0x08, 0xbc, 0x80, 0x6f, 0xeb, 0x13,
- 0xb1, 0xa6, 0x4f, 0x8b, 0xfd, 0x07, 0x16, 0x3a, 0x2c, 0x16, 0x55, 0x5d, 0xfd, 0xdd, 0xcd, 0x99,
- 0xa1, 0x25, 0x62, 0xb1, 0xd8, 0x1b, 0xa7, 0xde, 0x7b, 0xbf, 0xf7, 0xea, 0xd5, 0xab, 0xf7, 0x5e,
- 0x57, 0x15, 0xe1, 0xde, 0xde, 0x6d, 0xbb, 0xaa, 0x1a, 0xb5, 0x3d, 0x67, 0x9b, 0x58, 0x3a, 0xa1,
- 0xc4, 0xae, 0xed, 0x13, 0xbd, 0x65, 0x58, 0x35, 0x41, 0x50, 0x4c, 0xb5, 0x46, 0x0e, 0x28, 0xd1,
- 0x6d, 0xd5, 0xd0, 0xed, 0xda, 0xfe, 0xf5, 0x6d, 0x42, 0x95, 0xeb, 0xb5, 0x36, 0xd1, 0x89, 0xa5,
- 0x50, 0xd2, 0xaa, 0x9a, 0x96, 0x41, 0x0d, 0x74, 0xc1, 0x65, 0xaf, 0x2a, 0xa6, 0x5a, 0x0d, 0xd8,
- 0xab, 0x82, 0x7d, 0xee, 0x5a, 0x5b, 0xa5, 0xbb, 0xce, 0x76, 0xb5, 0x69, 0x74, 0x6a, 0x6d, 0xa3,
- 0x6d, 0xd4, 0xb8, 0xd4, 0xb6, 0xb3, 0xc3, 0x7f, 0xf1, 0x1f, 0xfc, 0x2f, 0x17, 0x6d, 0x4e, 0x0e,
- 0x29, 0x6f, 0x1a, 0x16, 0xa9, 0xed, 0x27, 0x34, 0xce, 0x5d, 0x09, 0xf1, 0x98, 0x86, 0xa6, 0x36,
- 0xbb, 0x59, 0xc6, 0xcd, 0xbd, 0x15, 0xb0, 0x76, 0x94, 0xe6, 0xae, 0xaa, 0x13, 0xab, 0x5b, 0x33,
- 0xf7, 0xda, 0x5c, 0xd6, 0x22, 0xb6, 0xe1, 0x58, 0x4d, 0x32, 0x90, 0x94, 0x5d, 0xeb, 0x10, 0xaa,
- 0xa4, 0x99, 0x55, 0xcb, 0x92, 0xb2, 0x1c, 0x9d, 0xaa, 0x9d, 0xa4, 0x9a, 0x5b, 0xbd, 0x04, 0xec,
- 0xe6, 0x2e, 0xe9, 0x28, 0x09, 0xb9, 0x37, 0xb3, 0xe4, 0x1c, 0xaa, 0x6a, 0x35, 0x55, 0xa7, 0x36,
- 0xb5, 0xe2, 0x42, 0xf2, 0x5d, 0x98, 0x58, 0xd0, 0x34, 0xe3, 0x53, 0xd2, 0x7a, 0x60, 0xd8, 0x74,
- 0x43, 0xa1, 0xbb, 0xe8, 0x06, 0x80, 0xa9, 0xd0, 0xdd, 0x0d, 0x8b, 0xec, 0xa8, 0x07, 0x65, 0xe9,
- 0xa2, 0x74, 0x79, 0xa4, 0x8e, 0x9e, 0x1e, 0x56, 0xce, 0x1c, 0x1d, 0x56, 0x60, 0xc3, 0xa7, 0xe0,
- 0x10, 0x97, 0xfc, 0x2f, 0x12, 0xbc, 0xbc, 0xe8, 0xd8, 0xd4, 0xe8, 0xac, 0x12, 0x6a, 0xa9, 0xcd,
- 0x45, 0xc7, 0xb2, 0x88, 0x4e, 0x1b, 0x54, 0xa1, 0x8e, 0x8d, 0x2e, 0x42, 0x41, 0x57, 0x3a, 0x44,
- 0x60, 0x9d, 0x15, 0x58, 0x85, 0x35, 0xa5, 0x43, 0x30, 0xa7, 0xa0, 0x0f, 0x61, 0x68, 0x5f, 0xd1,
- 0x1c, 0x52, 0xce, 0x5d, 0x94, 0x2e, 0x8f, 0xde, 0xa8, 0x56, 0x83, 0xe8, 0xf1, 0xe7, 0x52, 0x35,
- 0xf7, 0xda, 0x3c, 0x9c, 0xbc, 0x05, 0xaa, 0x3e, 0x74, 0x14, 0x9d, 0xaa, 0xb4, 0x5b, 0x3f, 0x27,
- 0x20, 0xcf, 0x0a, 0xbd, 0x8f, 0x18, 0x16, 0x76, 0x21, 0xe5, 0xbf, 0x80, 0x0b, 0x99, 0xa6, 0xad,
- 0xa8, 0x36, 0x45, 0x8f, 0x61, 0x48, 0xa5, 0xa4, 0x63, 0x97, 0xa5, 0x8b, 0xf9, 0xcb, 0xa3, 0x37,
- 0x6e, 0x57, 0x8f, 0x0d, 0xdd, 0x6a, 0x26, 0x58, 0x7d, 0x4c, 0x98, 0x31, 0xb4, 0xcc, 0xe0, 0xb0,
- 0x8b, 0x2a, 0xff, 0xa3, 0x04, 0x28, 0x2c, 0xb3, 0xa9, 0x58, 0x6d, 0x42, 0xfb, 0x70, 0xca, 0x9f,
- 0xfe, 0x38, 0xa7, 0x4c, 0x0b, 0xc8, 0x51, 0x57, 0x61, 0xc4, 0x27, 0x26, 0xcc, 0x24, 0x4d, 0xe2,
- 0xce, 0x78, 0x14, 0x75, 0xc6, 0xf5, 0x01, 0x9c, 0xe1, 0xa2, 0x64, 0x78, 0xe1, 0xb3, 0x1c, 0x8c,
- 0x2c, 0x29, 0xa4, 0x63, 0xe8, 0x0d, 0x42, 0xd1, 0xc7, 0x50, 0x62, 0xfb, 0xa5, 0xa5, 0x50, 0x85,
- 0x3b, 0x60, 0xf4, 0xc6, 0x1b, 0xc7, 0xcd, 0xce, 0xae, 0x32, 0xee, 0xea, 0xfe, 0xf5, 0xea, 0xfa,
- 0xf6, 0x13, 0xd2, 0xa4, 0xab, 0x84, 0x2a, 0x41, 0x4c, 0x06, 0x63, 0xd8, 0x47, 0x45, 0x6b, 0x50,
- 0xb0, 0x4d, 0xd2, 0x14, 0xbe, 0xbb, 0xda, 0x63, 0x1a, 0xbe, 0x65, 0x0d, 0x93, 0x34, 0x83, 0xc5,
- 0x60, 0xbf, 0x30, 0xc7, 0x41, 0x8f, 0x60, 0xd8, 0xe6, 0xab, 0x5c, 0xce, 0x27, 0x56, 0xe3, 0x78,
- 0x44, 0x37, 0x36, 0xc6, 0x05, 0xe6, 0xb0, 0xfb, 0x1b, 0x0b, 0x34, 0xf9, 0xe7, 0x39, 0x40, 0x3e,
- 0xef, 0xa2, 0xa1, 0xb7, 0x54, 0xaa, 0x1a, 0x3a, 0x7a, 0x07, 0x0a, 0xb4, 0x6b, 0x7a, 0xd1, 0x71,
- 0xc9, 0x33, 0x68, 0xb3, 0x6b, 0x92, 0x67, 0x87, 0x95, 0x99, 0xa4, 0x04, 0xa3, 0x60, 0x2e, 0x83,
- 0x56, 0x7c, 0x53, 0x73, 0x5c, 0xfa, 0xad, 0xa8, 0xea, 0x67, 0x87, 0x95, 0x94, 0x74, 0x5a, 0xf5,
- 0x91, 0xa2, 0x06, 0xa2, 0x7d, 0x40, 0x9a, 0x62, 0xd3, 0x4d, 0x4b, 0xd1, 0x6d, 0x57, 0x93, 0xda,
- 0x21, 0xc2, 0x09, 0xaf, 0xf7, 0xb7, 0x68, 0x4c, 0xa2, 0x3e, 0x27, 0xac, 0x40, 0x2b, 0x09, 0x34,
- 0x9c, 0xa2, 0x01, 0x5d, 0x82, 0x61, 0x8b, 0x28, 0xb6, 0xa1, 0x97, 0x0b, 0x7c, 0x16, 0xbe, 0x03,
- 0x31, 0x1f, 0xc5, 0x82, 0x8a, 0xae, 0x40, 0xb1, 0x43, 0x6c, 0x5b, 0x69, 0x93, 0xf2, 0x10, 0x67,
- 0x9c, 0x10, 0x8c, 0xc5, 0x55, 0x77, 0x18, 0x7b, 0x74, 0xf9, 0x4b, 0x09, 0xc6, 0x7c, 0xcf, 0xf1,
- 0x68, 0xff, 0xb3, 0x44, 0x1c, 0x56, 0xfb, 0x9b, 0x12, 0x93, 0xe6, 0x51, 0x38, 0x29, 0xb4, 0x95,
- 0xbc, 0x91, 0x50, 0x0c, 0xae, 0x7a, 0x7b, 0x29, 0xc7, 0xf7, 0xd2, 0xe5, 0x7e, 0x43, 0x26, 0x63,
- 0x0b, 0xfd, 0x53, 0x21, 0x64, 0x3e, 0x0b, 0x4d, 0xf4, 0x18, 0x4a, 0x36, 0xd1, 0x48, 0x93, 0x1a,
- 0x96, 0x30, 0xff, 0xcd, 0x3e, 0xcd, 0x57, 0xb6, 0x89, 0xd6, 0x10, 0xa2, 0xf5, 0xb3, 0xcc, 0x7e,
- 0xef, 0x17, 0xf6, 0x21, 0xd1, 0x43, 0x28, 0x51, 0xd2, 0x31, 0x35, 0x85, 0x7a, 0x39, 0xe8, 0xb5,
- 0xf0, 0x14, 0x58, 0xe4, 0x30, 0xb0, 0x0d, 0xa3, 0xb5, 0x29, 0xd8, 0xf8, 0xf6, 0xf1, 0x5d, 0xe2,
- 0x8d, 0x62, 0x1f, 0x06, 0xed, 0xc3, 0xb8, 0x63, 0xb6, 0x18, 0x27, 0x65, 0x65, 0xa8, 0xdd, 0x15,
- 0x91, 0x74, 0xab, 0x5f, 0xdf, 0x6c, 0x45, 0xa4, 0xeb, 0x33, 0x42, 0xd7, 0x78, 0x74, 0x1c, 0xc7,
- 0xb4, 0xa0, 0x05, 0x98, 0xe8, 0xa8, 0x3a, 0x26, 0x4a, 0xab, 0xdb, 0x20, 0x4d, 0x43, 0x6f, 0xd9,
- 0x3c, 0xac, 0x86, 0xea, 0xb3, 0x02, 0x60, 0x62, 0x35, 0x4a, 0xc6, 0x71, 0x7e, 0xf4, 0x3e, 0x20,
- 0x6f, 0x1a, 0xf7, 0xdd, 0x2a, 0xaa, 0x1a, 0x3a, 0x8f, 0xb9, 0x7c, 0x10, 0xdc, 0x9b, 0x09, 0x0e,
- 0x9c, 0x22, 0x85, 0x56, 0xe0, 0x9c, 0x45, 0xf6, 0x55, 0x36, 0xc7, 0x07, 0xaa, 0x4d, 0x0d, 0xab,
- 0xbb, 0xa2, 0x76, 0x54, 0x5a, 0x1e, 0xe6, 0x36, 0x95, 0x8f, 0x0e, 0x2b, 0xe7, 0x70, 0x0a, 0x1d,
- 0xa7, 0x4a, 0xc9, 0xff, 0x3c, 0x0c, 0x13, 0xb1, 0x7c, 0x83, 0x1e, 0xc1, 0x4c, 0xd3, 0x2d, 0x4e,
- 0x6b, 0x4e, 0x67, 0x9b, 0x58, 0x8d, 0xe6, 0x2e, 0x69, 0x39, 0x1a, 0x69, 0xf1, 0x40, 0x19, 0xaa,
- 0xcf, 0x0b, 0x8b, 0x67, 0x16, 0x53, 0xb9, 0x70, 0x86, 0x34, 0xf3, 0x82, 0xce, 0x87, 0x56, 0x55,
- 0xdb, 0xf6, 0x31, 0x73, 0x1c, 0xd3, 0xf7, 0xc2, 0x5a, 0x82, 0x03, 0xa7, 0x48, 0x31, 0x1b, 0x5b,
- 0xc4, 0x56, 0x2d, 0xd2, 0x8a, 0xdb, 0x98, 0x8f, 0xda, 0xb8, 0x94, 0xca, 0x85, 0x33, 0xa4, 0xd1,
- 0x4d, 0x18, 0x75, 0xb5, 0xf1, 0xf5, 0x13, 0x0b, 0xed, 0x97, 0xc3, 0xb5, 0x80, 0x84, 0xc3, 0x7c,
- 0x6c, 0x6a, 0xc6, 0xb6, 0x4d, 0xac, 0x7d, 0xd2, 0xca, 0x5e, 0xe0, 0xf5, 0x04, 0x07, 0x4e, 0x91,
- 0x62, 0x53, 0x73, 0x23, 0x30, 0x31, 0xb5, 0xe1, 0xe8, 0xd4, 0xb6, 0x52, 0xb9, 0x70, 0x86, 0x34,
- 0x8b, 0x63, 0xd7, 0xe4, 0x85, 0x7d, 0x45, 0xd5, 0x94, 0x6d, 0x8d, 0x94, 0x8b, 0xd1, 0x38, 0x5e,
- 0x8b, 0x92, 0x71, 0x9c, 0x1f, 0xdd, 0x87, 0x29, 0x77, 0x68, 0x4b, 0x57, 0x7c, 0x90, 0x12, 0x07,
- 0x79, 0x59, 0x80, 0x4c, 0xad, 0xc5, 0x19, 0x70, 0x52, 0x06, 0xbd, 0x03, 0xe3, 0x4d, 0x43, 0xd3,
- 0x78, 0x3c, 0x2e, 0x1a, 0x8e, 0x4e, 0xcb, 0x23, 0x1c, 0x05, 0xb1, 0xfd, 0xb8, 0x18, 0xa1, 0xe0,
- 0x18, 0x27, 0x22, 0x00, 0x4d, 0xaf, 0xe0, 0xd8, 0x65, 0xe8, 0xab, 0xd7, 0x48, 0x16, 0xbd, 0xa0,
- 0x07, 0xf0, 0x87, 0x6c, 0x1c, 0x02, 0x96, 0xff, 0x5f, 0x82, 0xd9, 0x8c, 0xd4, 0x81, 0xde, 0x8b,
- 0x94, 0xd8, 0xdf, 0x8f, 0x95, 0xd8, 0xf3, 0x19, 0x62, 0xa1, 0x3a, 0xab, 0xc3, 0x98, 0xc5, 0x66,
- 0xa5, 0xb7, 0x5d, 0x16, 0x91, 0x23, 0x6f, 0xf6, 0x98, 0x06, 0x0e, 0xcb, 0x04, 0x39, 0x7f, 0xea,
- 0xe8, 0xb0, 0x32, 0x16, 0xa1, 0xe1, 0x28, 0xbc, 0xfc, 0x79, 0x0e, 0x60, 0x89, 0x98, 0x9a, 0xd1,
- 0xed, 0x10, 0xfd, 0x34, 0x7a, 0xa8, 0xf5, 0x48, 0x0f, 0x75, 0xad, 0xd7, 0xf2, 0xf8, 0xa6, 0x65,
- 0x36, 0x51, 0x7f, 0x12, 0x6b, 0xa2, 0x6a, 0xfd, 0x43, 0x1e, 0xdf, 0x45, 0xfd, 0x34, 0x0f, 0xd3,
- 0x01, 0x73, 0xd0, 0x46, 0xdd, 0x89, 0xac, 0xf1, 0xef, 0xc5, 0xd6, 0x78, 0x36, 0x45, 0xe4, 0x85,
- 0xf5, 0x51, 0xcf, 0xbf, 0x9f, 0x41, 0x4f, 0x60, 0x9c, 0x35, 0x4e, 0x6e, 0x78, 0xf0, 0xb6, 0x6c,
- 0x78, 0xe0, 0xb6, 0xcc, 0x2f, 0xa0, 0x2b, 0x11, 0x24, 0x1c, 0x43, 0xce, 0x68, 0x03, 0x8b, 0x2f,
- 0xba, 0x0d, 0x94, 0xbf, 0x92, 0x60, 0x3c, 0x58, 0xa6, 0x53, 0x68, 0xda, 0xd6, 0xa2, 0x4d, 0xdb,
- 0x95, 0xbe, 0x43, 0x34, 0xa3, 0x6b, 0xfb, 0x25, 0x6b, 0xf0, 0x7d, 0x26, 0xb6, 0xc1, 0xb7, 0x95,
- 0xe6, 0x5e, 0x1f, 0x9f, 0x7f, 0x9f, 0x49, 0x80, 0x44, 0x15, 0x58, 0xd0, 0x75, 0x83, 0x2a, 0x6e,
- 0xae, 0x74, 0xcd, 0x5a, 0xee, 0xdb, 0x2c, 0x4f, 0x63, 0x75, 0x2b, 0x81, 0x75, 0x57, 0xa7, 0x56,
- 0x37, 0x58, 0x91, 0x24, 0x03, 0x4e, 0x31, 0x00, 0x29, 0x00, 0x96, 0xc0, 0xdc, 0x34, 0xc4, 0x46,
- 0xbe, 0xd6, 0x47, 0xce, 0x63, 0x02, 0x8b, 0x86, 0xbe, 0xa3, 0xb6, 0x83, 0xb4, 0x83, 0x7d, 0x20,
- 0x1c, 0x02, 0x9d, 0xbb, 0x0b, 0xb3, 0x19, 0xd6, 0xa2, 0x49, 0xc8, 0xef, 0x91, 0xae, 0xeb, 0x36,
- 0xcc, 0xfe, 0x44, 0xe7, 0xc2, 0x9f, 0xc9, 0x23, 0xe2, 0x0b, 0xf7, 0x9d, 0xdc, 0x6d, 0x49, 0xfe,
- 0x72, 0x28, 0x1c, 0x3b, 0xbc, 0x63, 0xbe, 0x0c, 0x25, 0x8b, 0x98, 0x9a, 0xda, 0x54, 0x6c, 0xd1,
- 0x08, 0xf1, 0xe6, 0x17, 0x8b, 0x31, 0xec, 0x53, 0x23, 0xbd, 0x75, 0xee, 0xc5, 0xf6, 0xd6, 0xf9,
- 0xe7, 0xd3, 0x5b, 0xff, 0x39, 0x94, 0x6c, 0xaf, 0xab, 0x2e, 0x70, 0xc8, 0xeb, 0x03, 0xe4, 0x57,
- 0xd1, 0x50, 0xfb, 0x0a, 0xfc, 0x56, 0xda, 0x07, 0x4d, 0x6b, 0xa2, 0x87, 0x06, 0x6c, 0xa2, 0x9f,
- 0x6b, 0xe3, 0xcb, 0x72, 0xaa, 0xa9, 0x38, 0x36, 0x69, 0xf1, 0x44, 0x54, 0x0a, 0x72, 0xea, 0x06,
- 0x1f, 0xc5, 0x82, 0x8a, 0x1e, 0x47, 0x42, 0xb6, 0x74, 0x92, 0x90, 0x1d, 0xcf, 0x0e, 0x57, 0xb4,
- 0x05, 0xb3, 0xa6, 0x65, 0xb4, 0x2d, 0x62, 0xdb, 0x4b, 0x44, 0x69, 0x69, 0xaa, 0x4e, 0x3c, 0xff,
- 0xb8, 0x1d, 0xd1, 0xf9, 0xa3, 0xc3, 0xca, 0xec, 0x46, 0x3a, 0x0b, 0xce, 0x92, 0x95, 0x9f, 0x16,
- 0x60, 0x32, 0x5e, 0x01, 0x33, 0x9a, 0x54, 0xe9, 0x44, 0x4d, 0xea, 0xd5, 0xd0, 0x66, 0x70, 0x3b,
- 0x78, 0x7f, 0xf5, 0x53, 0x36, 0xc4, 0x02, 0x4c, 0x88, 0x6c, 0xe0, 0x11, 0x45, 0x9b, 0xee, 0xaf,
- 0xfe, 0x56, 0x94, 0x8c, 0xe3, 0xfc, 0xac, 0xf5, 0x0c, 0x3a, 0x4a, 0x0f, 0xa4, 0x10, 0x6d, 0x3d,
- 0x17, 0xe2, 0x0c, 0x38, 0x29, 0x83, 0x56, 0x61, 0xda, 0xd1, 0x93, 0x50, 0x6e, 0x34, 0x9e, 0x17,
- 0x50, 0xd3, 0x5b, 0x49, 0x16, 0x9c, 0x26, 0x87, 0x76, 0x22, 0xdd, 0xe8, 0x30, 0xcf, 0xb0, 0x37,
- 0xfa, 0xde, 0x3b, 0x7d, 0xb7, 0xa3, 0xe8, 0x0e, 0x8c, 0x59, 0xfc, 0xbb, 0xc3, 0x33, 0xd8, 0xed,
- 0xdd, 0x5f, 0x12, 0x62, 0x63, 0x38, 0x4c, 0xc4, 0x51, 0xde, 0x94, 0x76, 0xbb, 0xd4, 0x6f, 0xbb,
- 0x2d, 0xff, 0xaf, 0x14, 0x2e, 0x42, 0x7e, 0x0b, 0xdc, 0xeb, 0x94, 0x29, 0x21, 0x11, 0xea, 0x8e,
- 0x8c, 0xf4, 0xee, 0xf7, 0xd6, 0x40, 0xdd, 0x6f, 0x50, 0x3c, 0x7b, 0xb7, 0xbf, 0x5f, 0x48, 0x30,
- 0x73, 0xaf, 0x71, 0xdf, 0x32, 0x1c, 0xd3, 0x33, 0x67, 0xdd, 0x74, 0xfd, 0xfa, 0x36, 0x14, 0x2c,
- 0x47, 0xf3, 0xe6, 0xf1, 0x9a, 0x37, 0x0f, 0xec, 0x68, 0x6c, 0x1e, 0xd3, 0x31, 0x29, 0x77, 0x12,
- 0x4c, 0x00, 0xad, 0xc1, 0xb0, 0xa5, 0xe8, 0x6d, 0xe2, 0x95, 0xd5, 0x4b, 0x3d, 0xac, 0x5f, 0x5e,
- 0xc2, 0x8c, 0x3d, 0xd4, 0xbc, 0x71, 0x69, 0x2c, 0x50, 0xe4, 0xbf, 0x93, 0x60, 0xe2, 0xc1, 0xe6,
- 0xe6, 0xc6, 0xb2, 0xce, 0x77, 0x34, 0x3f, 0x4f, 0xbf, 0x08, 0x05, 0x53, 0xa1, 0xbb, 0xf1, 0x4a,
- 0xcf, 0x68, 0x98, 0x53, 0xd0, 0x07, 0x50, 0x64, 0x99, 0x84, 0xe8, 0xad, 0x3e, 0x5b, 0x6d, 0x01,
- 0x5f, 0x77, 0x85, 0x82, 0x0e, 0x51, 0x0c, 0x60, 0x0f, 0x4e, 0xde, 0x83, 0x73, 0x21, 0x73, 0x98,
- 0x3f, 0xf8, 0x31, 0x30, 0x6a, 0xc0, 0x10, 0xd3, 0xec, 0x9d, 0xf2, 0xf6, 0x3a, 0xcc, 0x8c, 0x4d,
- 0x29, 0xe8, 0x74, 0xd8, 0x2f, 0x1b, 0xbb, 0x58, 0xf2, 0x2a, 0x8c, 0xf1, 0x4b, 0x04, 0xc3, 0xa2,
- 0xdc, 0x2d, 0xe8, 0x02, 0xe4, 0x3b, 0xaa, 0x2e, 0xea, 0xec, 0xa8, 0x90, 0xc9, 0xb3, 0x1a, 0xc1,
- 0xc6, 0x39, 0x59, 0x39, 0x10, 0x99, 0x27, 0x20, 0x2b, 0x07, 0x98, 0x8d, 0xcb, 0xf7, 0xa1, 0x28,
- 0xdc, 0x1d, 0x06, 0xca, 0x1f, 0x0f, 0x94, 0x4f, 0x01, 0x5a, 0x87, 0xe2, 0xf2, 0x46, 0x5d, 0x33,
- 0xdc, 0xae, 0xab, 0xa9, 0xb6, 0xac, 0xf8, 0x5a, 0x2c, 0x2e, 0x2f, 0x61, 0xcc, 0x29, 0x48, 0x86,
- 0x61, 0x72, 0xd0, 0x24, 0x26, 0xe5, 0x11, 0x31, 0x52, 0x07, 0xb6, 0xca, 0x77, 0xf9, 0x08, 0x16,
- 0x14, 0xf9, 0xef, 0x73, 0x50, 0x14, 0xee, 0x38, 0x85, 0xaf, 0xb0, 0x95, 0xc8, 0x57, 0xd8, 0xeb,
- 0xfd, 0x85, 0x46, 0xe6, 0x27, 0xd8, 0x66, 0xec, 0x13, 0xec, 0x6a, 0x9f, 0x78, 0xc7, 0x7f, 0x7f,
- 0xfd, 0x97, 0x04, 0xe3, 0xd1, 0xa0, 0x44, 0x37, 0x61, 0x94, 0x15, 0x1c, 0xb5, 0x49, 0xd6, 0x82,
- 0x3e, 0xd7, 0x3f, 0x84, 0x69, 0x04, 0x24, 0x1c, 0xe6, 0x43, 0x6d, 0x5f, 0x8c, 0xc5, 0x91, 0x98,
- 0x74, 0xb6, 0x4b, 0x1d, 0xaa, 0x6a, 0x55, 0xf7, 0x6e, 0xab, 0xba, 0xac, 0xd3, 0x75, 0xab, 0x41,
- 0x2d, 0x55, 0x6f, 0x27, 0x14, 0xf1, 0xa0, 0x0c, 0x23, 0xcb, 0xff, 0x23, 0xc1, 0xa8, 0x30, 0xf9,
- 0x14, 0xbe, 0x2a, 0xfe, 0x38, 0xfa, 0x55, 0x71, 0xa9, 0xcf, 0x0d, 0x9e, 0xfe, 0x49, 0xf1, 0x6f,
- 0x81, 0xe9, 0x6c, 0x4b, 0xb3, 0xa8, 0xde, 0x35, 0x6c, 0x1a, 0x8f, 0x6a, 0xb6, 0x19, 0x31, 0xa7,
- 0x20, 0x07, 0x26, 0xd5, 0x58, 0x0e, 0x10, 0xae, 0xad, 0xf5, 0x67, 0x89, 0x2f, 0x56, 0x2f, 0x0b,
- 0xf8, 0xc9, 0x38, 0x05, 0x27, 0x54, 0xc8, 0x04, 0x12, 0x5c, 0xe8, 0x21, 0x14, 0x76, 0x29, 0x35,
- 0x53, 0xce, 0xab, 0x7b, 0x64, 0x9e, 0xc0, 0x84, 0x12, 0x9f, 0xdd, 0xe6, 0xe6, 0x06, 0xe6, 0x50,
- 0xf2, 0xaf, 0x02, 0x7f, 0x34, 0xdc, 0x18, 0xf7, 0xf3, 0xa9, 0x74, 0x92, 0x7c, 0x3a, 0x9a, 0x96,
- 0x4b, 0xd1, 0x03, 0xc8, 0x53, 0xad, 0xdf, 0xcf, 0x42, 0x81, 0xb8, 0xb9, 0xd2, 0x08, 0x12, 0xd2,
- 0xe6, 0x4a, 0x03, 0x33, 0x08, 0xb4, 0x0e, 0x43, 0xac, 0xfa, 0xb0, 0x2d, 0x98, 0xef, 0x7f, 0x4b,
- 0xb3, 0xf9, 0x07, 0x01, 0xc1, 0x7e, 0xd9, 0xd8, 0xc5, 0x91, 0x3f, 0x81, 0xb1, 0xc8, 0x3e, 0x45,
- 0x1f, 0xc3, 0x59, 0xcd, 0x50, 0x5a, 0x75, 0x45, 0x53, 0xf4, 0x26, 0xf1, 0x2e, 0x07, 0x2e, 0xa5,
- 0x7d, 0x61, 0xac, 0x84, 0xf8, 0xc4, 0x2e, 0xf7, 0xaf, 0x53, 0xc3, 0x34, 0x1c, 0x41, 0x94, 0x15,
- 0x80, 0x60, 0x8e, 0xa8, 0x02, 0x43, 0x2c, 0xce, 0xdc, 0x7a, 0x32, 0x52, 0x1f, 0x61, 0x16, 0xb2,
- 0xf0, 0xb3, 0xb1, 0x3b, 0x8e, 0x6e, 0x00, 0xd8, 0xa4, 0x69, 0x11, 0xca, 0x93, 0x41, 0x2e, 0x7a,
- 0xa9, 0xdc, 0xf0, 0x29, 0x38, 0xc4, 0x25, 0xff, 0x9f, 0x04, 0x63, 0x6b, 0x84, 0x7e, 0x6a, 0x58,
- 0x7b, 0x1b, 0xfc, 0x3e, 0xff, 0x14, 0x92, 0x2d, 0x8e, 0x24, 0xdb, 0x37, 0x7a, 0xac, 0x4c, 0xc4,
- 0xba, 0xac, 0x94, 0x2b, 0x7f, 0x25, 0xc1, 0x6c, 0x84, 0xf3, 0x6e, 0xb0, 0x75, 0xb7, 0x60, 0xc8,
- 0x34, 0x2c, 0xea, 0x15, 0xe2, 0x81, 0x14, 0xb2, 0x34, 0x16, 0x2a, 0xc5, 0x0c, 0x06, 0xbb, 0x68,
- 0x68, 0x05, 0x72, 0xd4, 0x10, 0xa1, 0x3a, 0x18, 0x26, 0x21, 0x56, 0x1d, 0x04, 0x66, 0x6e, 0xd3,
- 0xc0, 0x39, 0x6a, 0xb0, 0x85, 0x28, 0x47, 0xb8, 0xc2, 0xc9, 0xe7, 0x05, 0xcd, 0x00, 0x43, 0x61,
- 0xc7, 0x32, 0x3a, 0x27, 0x9e, 0x83, 0xbf, 0x10, 0xf7, 0x2c, 0xa3, 0x83, 0x39, 0x96, 0xfc, 0xb5,
- 0x04, 0x53, 0x11, 0xce, 0x53, 0x48, 0xfc, 0x0f, 0xa3, 0x89, 0xff, 0xea, 0x20, 0x13, 0xc9, 0x48,
- 0xff, 0x5f, 0xe7, 0x62, 0xd3, 0x60, 0x13, 0x46, 0x3b, 0x30, 0x6a, 0x1a, 0xad, 0xc6, 0x73, 0xb8,
- 0x0e, 0x9c, 0x60, 0x75, 0x73, 0x23, 0xc0, 0xc2, 0x61, 0x60, 0x74, 0x00, 0x53, 0xba, 0xd2, 0x21,
- 0xb6, 0xa9, 0x34, 0x49, 0xe3, 0x39, 0x1c, 0x90, 0xbc, 0xc4, 0xef, 0x1b, 0xe2, 0x88, 0x38, 0xa9,
- 0x04, 0xad, 0x42, 0x51, 0x35, 0x79, 0x1f, 0x27, 0x7a, 0x97, 0x9e, 0x55, 0xd4, 0xed, 0xfa, 0xdc,
- 0x7c, 0x2e, 0x7e, 0x60, 0x0f, 0x43, 0xfe, 0xf7, 0x78, 0x34, 0xb0, 0xf8, 0x43, 0xf7, 0xa1, 0xc4,
- 0x5f, 0xc6, 0x34, 0x0d, 0xcd, 0xbb, 0x19, 0x60, 0x2b, 0xbb, 0x21, 0xc6, 0x9e, 0x1d, 0x56, 0xce,
- 0xa7, 0x1c, 0xfa, 0x7a, 0x64, 0xec, 0x0b, 0xa3, 0x35, 0x28, 0x98, 0x3f, 0xa6, 0x83, 0xe1, 0x45,
- 0x8e, 0xb7, 0x2d, 0x1c, 0x47, 0xfe, 0xab, 0x7c, 0xcc, 0x5c, 0x5e, 0xea, 0x9e, 0x3c, 0xb7, 0x55,
- 0xf7, 0x3b, 0xa6, 0xcc, 0x95, 0xdf, 0x86, 0xa2, 0xa8, 0xf0, 0x22, 0x98, 0xdf, 0x1e, 0x24, 0x98,
- 0xc3, 0x55, 0xcc, 0xff, 0x60, 0xf1, 0x06, 0x3d, 0x60, 0xf4, 0x11, 0x0c, 0x13, 0x57, 0x85, 0x5b,
- 0x1b, 0x6f, 0x0d, 0xa2, 0x22, 0xc8, 0xab, 0x41, 0xa3, 0x2a, 0xc6, 0x04, 0x2a, 0x7a, 0x8f, 0xf9,
- 0x8b, 0xf1, 0xb2, 0x8f, 0x40, 0xbb, 0x5c, 0xe0, 0xe5, 0xea, 0x82, 0x3b, 0x6d, 0x7f, 0xf8, 0xd9,
- 0x61, 0x05, 0x82, 0x9f, 0x38, 0x2c, 0x21, 0xff, 0x44, 0x82, 0x29, 0xee, 0xa1, 0xa6, 0x63, 0xa9,
- 0xb4, 0x7b, 0x6a, 0x85, 0xe9, 0x51, 0xa4, 0x30, 0xbd, 0xd5, 0xc3, 0x2d, 0x09, 0x0b, 0x33, 0x8b,
- 0xd3, 0x37, 0x12, 0xbc, 0x94, 0xe0, 0x3e, 0x85, 0xbc, 0xb8, 0x15, 0xcd, 0x8b, 0x6f, 0x0c, 0x3a,
- 0xa1, 0x8c, 0xdc, 0xf8, 0xf9, 0xd9, 0x94, 0xe9, 0xf0, 0x9d, 0x72, 0x03, 0xc0, 0xb4, 0xd4, 0x7d,
- 0x55, 0x23, 0x6d, 0x71, 0x09, 0x5e, 0x0a, 0x3d, 0x6b, 0xf3, 0x29, 0x38, 0xc4, 0x85, 0x6c, 0x98,
- 0x69, 0x91, 0x1d, 0xc5, 0xd1, 0xe8, 0x42, 0xab, 0xb5, 0xa8, 0x98, 0xca, 0xb6, 0xaa, 0xa9, 0x54,
- 0x15, 0xc7, 0x05, 0x23, 0xf5, 0x3b, 0xee, 0xe5, 0x74, 0x1a, 0xc7, 0xb3, 0xc3, 0xca, 0x85, 0xb4,
- 0xdb, 0x21, 0x8f, 0xa5, 0x8b, 0x33, 0xa0, 0x51, 0x17, 0xca, 0x16, 0xf9, 0xc4, 0x51, 0x2d, 0xd2,
- 0x5a, 0xb2, 0x0c, 0x33, 0xa2, 0x36, 0xcf, 0xd5, 0xfe, 0xe1, 0xd1, 0x61, 0xa5, 0x8c, 0x33, 0x78,
- 0x7a, 0x2b, 0xce, 0x84, 0x47, 0x4f, 0x60, 0x5a, 0x71, 0x5f, 0x03, 0x46, 0xb4, 0xba, 0xbb, 0xe4,
- 0xf6, 0xd1, 0x61, 0x65, 0x7a, 0x21, 0x49, 0xee, 0xad, 0x30, 0x0d, 0x14, 0xd5, 0xa0, 0xb8, 0x6f,
- 0x68, 0x4e, 0x87, 0xd8, 0xe5, 0x21, 0x8e, 0xcf, 0x0a, 0x41, 0xf1, 0x91, 0x3b, 0xf4, 0xec, 0xb0,
- 0x32, 0x7c, 0xaf, 0xc1, 0x77, 0x9f, 0xc7, 0xc5, 0x3e, 0x28, 0x59, 0x2f, 0x29, 0x76, 0x3c, 0x3f,
- 0x31, 0x2e, 0x05, 0x59, 0xeb, 0x41, 0x40, 0xc2, 0x61, 0x3e, 0xf4, 0x18, 0x46, 0x76, 0xc5, 0xa9,
- 0x84, 0x5d, 0x2e, 0xf6, 0x55, 0x84, 0x23, 0xa7, 0x18, 0xf5, 0x29, 0xa1, 0x62, 0xc4, 0x1b, 0xb6,
- 0x71, 0x80, 0x88, 0xae, 0x40, 0x91, 0xff, 0x58, 0x5e, 0xe2, 0xc7, 0x71, 0xa5, 0x20, 0xb7, 0x3d,
- 0x70, 0x87, 0xb1, 0x47, 0xf7, 0x58, 0x97, 0x37, 0x16, 0xf9, 0xb1, 0x70, 0x8c, 0x75, 0x79, 0x63,
- 0x11, 0x7b, 0x74, 0xf4, 0x31, 0x14, 0x6d, 0xb2, 0xa2, 0xea, 0xce, 0x41, 0x19, 0xfa, 0xba, 0x54,
- 0x6e, 0xdc, 0xe5, 0xdc, 0xb1, 0x83, 0xb1, 0x40, 0x83, 0xa0, 0x63, 0x0f, 0x16, 0xed, 0xc2, 0x88,
- 0xe5, 0xe8, 0x0b, 0xf6, 0x96, 0x4d, 0xac, 0xf2, 0x28, 0xd7, 0xd1, 0x2b, 0x9d, 0x63, 0x8f, 0x3f,
- 0xae, 0xc5, 0xf7, 0x90, 0xcf, 0x81, 0x03, 0x70, 0xf4, 0xb7, 0x12, 0x20, 0xdb, 0x31, 0x4d, 0x8d,
- 0x74, 0x88, 0x4e, 0x15, 0x8d, 0x9f, 0xc5, 0xd9, 0xe5, 0xb3, 0x5c, 0xe7, 0x1f, 0xf5, 0x9a, 0x57,
- 0x42, 0x30, 0xae, 0xdc, 0x3f, 0xf4, 0x4e, 0xb2, 0xe2, 0x14, 0xbd, 0xcc, 0xb5, 0x3b, 0x36, 0xff,
- 0xbb, 0x3c, 0xd6, 0x97, 0x6b, 0xd3, 0xcf, 0x1c, 0x03, 0xd7, 0x0a, 0x3a, 0xf6, 0x60, 0xd1, 0x23,
- 0x98, 0xb1, 0x88, 0xd2, 0x5a, 0xd7, 0xb5, 0x2e, 0x36, 0x0c, 0x7a, 0x4f, 0xd5, 0x88, 0xdd, 0xb5,
- 0x29, 0xe9, 0x94, 0xc7, 0xf9, 0xb2, 0xfb, 0x6f, 0x3f, 0x70, 0x2a, 0x17, 0xce, 0x90, 0x46, 0x1d,
- 0xa8, 0x78, 0x29, 0x83, 0xed, 0x27, 0x3f, 0x67, 0xdd, 0xb5, 0x9b, 0x8a, 0xe6, 0xde, 0x03, 0x4c,
- 0x70, 0x05, 0xaf, 0x1d, 0x1d, 0x56, 0x2a, 0x4b, 0xc7, 0xb3, 0xe2, 0x5e, 0x58, 0xe8, 0x03, 0x28,
- 0x2b, 0x59, 0x7a, 0x26, 0xb9, 0x9e, 0x57, 0x58, 0x1e, 0xca, 0x54, 0x90, 0x29, 0x8d, 0x28, 0x4c,
- 0x2a, 0xd1, 0x47, 0xc7, 0x76, 0x79, 0xaa, 0xaf, 0x83, 0xc8, 0xd8, 0x5b, 0xe5, 0xe0, 0x30, 0x22,
- 0x46, 0xb0, 0x71, 0x42, 0x03, 0x7f, 0x3e, 0x21, 0x0e, 0xd3, 0x4f, 0xe7, 0x09, 0xea, 0x60, 0xcf,
- 0x27, 0x02, 0xd3, 0x9e, 0xdb, 0xf3, 0x89, 0x10, 0xe4, 0xf1, 0xc7, 0x77, 0xbf, 0xc8, 0xc1, 0x74,
- 0xc0, 0xdc, 0xf7, 0xf3, 0x89, 0x14, 0x91, 0xdf, 0x3d, 0x43, 0xed, 0xfd, 0x0c, 0xf5, 0x2b, 0x09,
- 0xc6, 0x03, 0xd7, 0xfd, 0xe6, 0x3d, 0x69, 0x08, 0x6c, 0xcb, 0x68, 0xb2, 0xfe, 0x33, 0x17, 0x9e,
- 0xc0, 0x6f, 0xfd, 0xbd, 0xfa, 0x8f, 0x7f, 0x3b, 0x2a, 0x7f, 0x93, 0x87, 0xc9, 0xf8, 0x6e, 0x8c,
- 0x5c, 0xbf, 0x4a, 0x3d, 0xaf, 0x5f, 0x37, 0xe0, 0xdc, 0x8e, 0xa3, 0x69, 0x5d, 0xee, 0x86, 0xd0,
- 0x1d, 0xac, 0x7b, 0x7d, 0xf2, 0x8a, 0x90, 0x3c, 0x77, 0x2f, 0x85, 0x07, 0xa7, 0x4a, 0x66, 0x5c,
- 0x25, 0xe7, 0x4f, 0x74, 0x95, 0x9c, 0xb8, 0xd9, 0x2c, 0x0c, 0x70, 0xb3, 0x99, 0x7a, 0x2d, 0x3c,
- 0x74, 0x82, 0x6b, 0xe1, 0x93, 0xdc, 0xe3, 0xa6, 0x24, 0xb1, 0x9e, 0xcf, 0x0a, 0x5f, 0x81, 0x39,
- 0x21, 0x46, 0xf9, 0x15, 0xab, 0x4e, 0x2d, 0x43, 0xd3, 0x88, 0xb5, 0xe4, 0x74, 0x3a, 0x5d, 0xf9,
- 0x5d, 0x18, 0x8f, 0x3e, 0x1e, 0x70, 0x57, 0xda, 0x7d, 0xbf, 0x20, 0x2e, 0xb1, 0x42, 0x2b, 0xed,
- 0x8e, 0x63, 0x9f, 0x43, 0xfe, 0x6b, 0x09, 0x66, 0xd2, 0x1f, 0x09, 0x22, 0x0d, 0xc6, 0x3b, 0xca,
- 0x41, 0xf8, 0xe1, 0xa6, 0x74, 0xc2, 0xe3, 0x05, 0x7e, 0x6b, 0xbc, 0x1a, 0xc1, 0xc2, 0x31, 0x6c,
- 0xf9, 0x07, 0x09, 0x66, 0x33, 0xee, 0x6b, 0x4f, 0xd7, 0x12, 0xf4, 0x21, 0x94, 0x3a, 0xca, 0x41,
- 0xc3, 0xb1, 0xda, 0xe4, 0xc4, 0x07, 0x2a, 0x3c, 0x63, 0xac, 0x0a, 0x14, 0xec, 0xe3, 0xc9, 0x5f,
- 0x48, 0x50, 0xce, 0x6a, 0x6d, 0xd1, 0xcd, 0xc8, 0xcd, 0xf2, 0xab, 0xb1, 0x9b, 0xe5, 0xa9, 0x84,
- 0xdc, 0x0b, 0xba, 0x57, 0xfe, 0x0f, 0x09, 0x66, 0xd2, 0x5b, 0x7c, 0xf4, 0x66, 0xc4, 0xc2, 0x4a,
- 0xcc, 0xc2, 0x89, 0x98, 0x94, 0xb0, 0xef, 0x23, 0x18, 0x17, 0x1f, 0x02, 0x02, 0x46, 0x78, 0x55,
- 0x4e, 0xcb, 0x95, 0x02, 0xc2, 0x6b, 0x7c, 0xf9, 0x7a, 0x45, 0xc7, 0x70, 0x0c, 0x4d, 0xfe, 0x9b,
- 0x1c, 0x0c, 0x35, 0x9a, 0x8a, 0x46, 0x4e, 0xa1, 0xcd, 0x7a, 0x3f, 0xd2, 0x66, 0xf5, 0xfa, 0x27,
- 0x0b, 0x6e, 0x55, 0x66, 0x87, 0x85, 0x63, 0x1d, 0xd6, 0xeb, 0x7d, 0xa1, 0x1d, 0xdf, 0x5c, 0xfd,
- 0x01, 0x8c, 0xf8, 0x4a, 0x07, 0xcb, 0xf9, 0xf2, 0xbf, 0xe6, 0x60, 0x34, 0xa4, 0x62, 0xc0, 0x8a,
- 0xb1, 0x13, 0xa9, 0xb4, 0xfd, 0xfc, 0x6b, 0x5b, 0x48, 0x57, 0xd5, 0xab, 0xad, 0xee, 0x23, 0xc1,
- 0xe0, 0x59, 0x58, 0xb2, 0xe4, 0xbe, 0x0b, 0xe3, 0x94, 0xff, 0xeb, 0x97, 0x7f, 0x0c, 0x99, 0xe7,
- 0xb1, 0xe8, 0x3f, 0x2d, 0xdd, 0x8c, 0x50, 0x71, 0x8c, 0x7b, 0xee, 0x0e, 0x8c, 0x45, 0x94, 0x0d,
- 0xf4, 0xc6, 0xef, 0xbf, 0x25, 0x78, 0xb5, 0xe7, 0x47, 0x22, 0xaa, 0x47, 0x36, 0x49, 0x35, 0xb6,
- 0x49, 0xe6, 0xb3, 0x01, 0x5e, 0xdc, 0x5b, 0x91, 0xfa, 0xb5, 0xa7, 0xdf, 0xcf, 0x9f, 0xf9, 0xf6,
- 0xfb, 0xf9, 0x33, 0xdf, 0x7d, 0x3f, 0x7f, 0xe6, 0x2f, 0x8f, 0xe6, 0xa5, 0xa7, 0x47, 0xf3, 0xd2,
- 0xb7, 0x47, 0xf3, 0xd2, 0x77, 0x47, 0xf3, 0xd2, 0xcf, 0x8e, 0xe6, 0xa5, 0x7f, 0xf8, 0x61, 0xfe,
- 0xcc, 0x87, 0x45, 0x01, 0xf7, 0xeb, 0x00, 0x00, 0x00, 0xff, 0xff, 0x22, 0xba, 0x16, 0x0c, 0x99,
- 0x3b, 0x00, 0x00,
+ 0x4d, 0x73, 0x80, 0x20, 0xc8, 0x21, 0x08, 0x10, 0x20, 0x41, 0x92, 0x83, 0xf3, 0x71, 0x8b, 0x2f,
+ 0x39, 0x25, 0x48, 0x6e, 0xc9, 0xc1, 0x30, 0x10, 0xc0, 0x0b, 0x08, 0x0b, 0x2f, 0xe0, 0xdb, 0xfa,
+ 0x44, 0xac, 0xe9, 0xd3, 0x62, 0xff, 0x81, 0x85, 0x0e, 0x8b, 0x45, 0x55, 0x57, 0x7f, 0x77, 0x73,
+ 0x66, 0x68, 0x89, 0x58, 0x2c, 0xf6, 0xc6, 0xa9, 0xf7, 0xde, 0xef, 0xbd, 0x7a, 0xf5, 0xea, 0xbd,
+ 0xd7, 0x55, 0x45, 0xb8, 0xb7, 0x77, 0xdb, 0xae, 0xaa, 0x46, 0x6d, 0xcf, 0xd9, 0x26, 0x96, 0x4e,
+ 0x28, 0xb1, 0x6b, 0xfb, 0x44, 0x6f, 0x19, 0x56, 0x4d, 0x10, 0x14, 0x53, 0xad, 0x91, 0x03, 0x4a,
+ 0x74, 0x5b, 0x35, 0x74, 0xbb, 0xb6, 0x7f, 0x7d, 0x9b, 0x50, 0xe5, 0x7a, 0xad, 0x4d, 0x74, 0x62,
+ 0x29, 0x94, 0xb4, 0xaa, 0xa6, 0x65, 0x50, 0x03, 0x5d, 0x70, 0xd9, 0xab, 0x8a, 0xa9, 0x56, 0x03,
+ 0xf6, 0xaa, 0x60, 0x9f, 0xbb, 0xd6, 0x56, 0xe9, 0xae, 0xb3, 0x5d, 0x6d, 0x1a, 0x9d, 0x5a, 0xdb,
+ 0x68, 0x1b, 0x35, 0x2e, 0xb5, 0xed, 0xec, 0xf0, 0x5f, 0xfc, 0x07, 0xff, 0xcb, 0x45, 0x9b, 0x93,
+ 0x43, 0xca, 0x9b, 0x86, 0x45, 0x6a, 0xfb, 0x09, 0x8d, 0x73, 0x57, 0x42, 0x3c, 0xa6, 0xa1, 0xa9,
+ 0xcd, 0x6e, 0x96, 0x71, 0x73, 0x6f, 0x05, 0xac, 0x1d, 0xa5, 0xb9, 0xab, 0xea, 0xc4, 0xea, 0xd6,
+ 0xcc, 0xbd, 0x36, 0x97, 0xb5, 0x88, 0x6d, 0x38, 0x56, 0x93, 0x0c, 0x24, 0x65, 0xd7, 0x3a, 0x84,
+ 0x2a, 0x69, 0x66, 0xd5, 0xb2, 0xa4, 0x2c, 0x47, 0xa7, 0x6a, 0x27, 0xa9, 0xe6, 0x56, 0x2f, 0x01,
+ 0xbb, 0xb9, 0x4b, 0x3a, 0x4a, 0x42, 0xee, 0xcd, 0x2c, 0x39, 0x87, 0xaa, 0x5a, 0x4d, 0xd5, 0xa9,
+ 0x4d, 0xad, 0xb8, 0x90, 0x7c, 0x07, 0xa6, 0x16, 0x34, 0xcd, 0xf8, 0x94, 0xb4, 0xee, 0x69, 0xe4,
+ 0xe0, 0x91, 0xa1, 0x39, 0x1d, 0x82, 0x2e, 0xc1, 0x70, 0xcb, 0x52, 0xf7, 0x89, 0x55, 0x96, 0x2e,
+ 0x4a, 0x97, 0x47, 0xea, 0xe3, 0x4f, 0x0f, 0x2b, 0x67, 0x8e, 0x0e, 0x2b, 0xc3, 0x4b, 0x7c, 0x14,
+ 0x0b, 0xaa, 0x7c, 0x17, 0x26, 0x84, 0xf0, 0x03, 0xc3, 0xa6, 0x1b, 0x0a, 0xdd, 0x45, 0x37, 0x00,
+ 0x4c, 0x85, 0xee, 0x6e, 0x58, 0x64, 0x47, 0x3d, 0x10, 0xe2, 0x48, 0x88, 0xc3, 0x86, 0x4f, 0xc1,
+ 0x21, 0x2e, 0xf9, 0xdf, 0x24, 0x78, 0x79, 0xd1, 0xb1, 0xa9, 0xd1, 0x59, 0x25, 0xd4, 0x52, 0x9b,
+ 0x8b, 0x8e, 0x65, 0x11, 0x9d, 0x36, 0xa8, 0x42, 0x1d, 0x1b, 0x5d, 0x84, 0x82, 0xae, 0x74, 0x88,
+ 0xc0, 0x3a, 0x2b, 0xb0, 0x0a, 0x6b, 0x4a, 0x87, 0x60, 0x4e, 0x41, 0x1f, 0xc2, 0xd0, 0xbe, 0xa2,
+ 0x39, 0xa4, 0x9c, 0xbb, 0x28, 0x5d, 0x1e, 0xbd, 0x51, 0xad, 0x06, 0xa1, 0xe7, 0x3b, 0xa2, 0x6a,
+ 0xee, 0xb5, 0x79, 0x2c, 0x7a, 0xab, 0x5b, 0x7d, 0xe8, 0x28, 0x3a, 0x55, 0x69, 0xb7, 0x7e, 0x4e,
+ 0x40, 0x9e, 0x15, 0x7a, 0x1f, 0x31, 0x2c, 0xec, 0x42, 0xca, 0x7f, 0x09, 0x17, 0x32, 0x4d, 0x5b,
+ 0x51, 0x6d, 0x8a, 0x1e, 0xc3, 0x90, 0x4a, 0x49, 0xc7, 0x2e, 0x4b, 0x17, 0xf3, 0x97, 0x47, 0x6f,
+ 0xdc, 0xae, 0x1e, 0x1b, 0xf7, 0xd5, 0x4c, 0xb0, 0xfa, 0x98, 0x30, 0x63, 0x68, 0x99, 0xc1, 0x61,
+ 0x17, 0x55, 0xfe, 0x27, 0x09, 0x50, 0x58, 0x66, 0x53, 0xb1, 0xda, 0x84, 0xf6, 0xe1, 0x94, 0x3f,
+ 0xfd, 0x61, 0x4e, 0x99, 0x16, 0x90, 0xa3, 0xae, 0xc2, 0x88, 0x4f, 0x4c, 0x98, 0x49, 0x9a, 0xc4,
+ 0x9d, 0xf1, 0x28, 0xea, 0x8c, 0xeb, 0x03, 0x38, 0xc3, 0x45, 0xc9, 0xf0, 0xc2, 0x67, 0x39, 0x18,
+ 0x59, 0x52, 0x48, 0xc7, 0xd0, 0x1b, 0x84, 0xa2, 0x8f, 0xa1, 0xc4, 0x36, 0x5b, 0x4b, 0xa1, 0x0a,
+ 0x77, 0xc0, 0xe8, 0x8d, 0x37, 0x8e, 0x9b, 0x9d, 0x5d, 0x65, 0xdc, 0xd5, 0xfd, 0xeb, 0xd5, 0xf5,
+ 0xed, 0x27, 0xa4, 0x49, 0x57, 0x09, 0x55, 0x82, 0x98, 0x0c, 0xc6, 0xb0, 0x8f, 0x8a, 0xd6, 0xa0,
+ 0x60, 0x9b, 0xa4, 0x29, 0x7c, 0x77, 0xb5, 0xc7, 0x34, 0x7c, 0xcb, 0x1a, 0x26, 0x69, 0x06, 0x8b,
+ 0xc1, 0x7e, 0x61, 0x8e, 0x83, 0x1e, 0xc1, 0xb0, 0xcd, 0x57, 0xb9, 0x9c, 0x4f, 0xac, 0xc6, 0xf1,
+ 0x88, 0x6e, 0x6c, 0xf8, 0x1b, 0xd0, 0xfd, 0x8d, 0x05, 0x9a, 0xfc, 0xf3, 0x1c, 0x20, 0x9f, 0x77,
+ 0xd1, 0xd0, 0x5b, 0x2a, 0x55, 0x0d, 0x1d, 0xbd, 0x03, 0x05, 0xda, 0x35, 0xbd, 0xe8, 0xb8, 0xe4,
+ 0x19, 0xb4, 0xd9, 0x35, 0xc9, 0xb3, 0xc3, 0xca, 0x4c, 0x52, 0x82, 0x51, 0x30, 0x97, 0x41, 0x2b,
+ 0xbe, 0xa9, 0x39, 0x2e, 0xfd, 0x56, 0x54, 0xf5, 0xb3, 0xc3, 0x4a, 0x4a, 0x2e, 0xae, 0xfa, 0x48,
+ 0x51, 0x03, 0xd1, 0x3e, 0x20, 0x4d, 0xb1, 0xe9, 0xa6, 0xa5, 0xe8, 0xb6, 0xab, 0x49, 0xed, 0x10,
+ 0xe1, 0x84, 0xd7, 0xfb, 0x5b, 0x34, 0x26, 0x51, 0x9f, 0x13, 0x56, 0xa0, 0x95, 0x04, 0x1a, 0x4e,
+ 0xd1, 0xc0, 0x32, 0x98, 0x45, 0x14, 0xdb, 0xd0, 0xcb, 0x85, 0x68, 0x06, 0xc3, 0x7c, 0x14, 0x0b,
+ 0x2a, 0xba, 0x02, 0xc5, 0x0e, 0xb1, 0x6d, 0xa5, 0x4d, 0xca, 0x43, 0x9c, 0x71, 0x42, 0x30, 0x16,
+ 0x57, 0xdd, 0x61, 0xec, 0xd1, 0xe5, 0x2f, 0x24, 0x18, 0xf3, 0x3d, 0xc7, 0xa3, 0xfd, 0xcf, 0x12,
+ 0x71, 0x58, 0xed, 0x6f, 0x4a, 0x4c, 0x9a, 0x47, 0xe1, 0xa4, 0xd0, 0x56, 0xf2, 0x46, 0x42, 0x31,
+ 0xb8, 0xea, 0xed, 0xa5, 0x1c, 0xdf, 0x4b, 0x97, 0xfb, 0x0d, 0x99, 0x8c, 0x2d, 0xf4, 0xcf, 0x85,
+ 0x90, 0xf9, 0x2c, 0x34, 0xd1, 0x63, 0x28, 0xd9, 0x44, 0x23, 0x4d, 0x6a, 0x58, 0xc2, 0xfc, 0x37,
+ 0xfb, 0x34, 0x5f, 0xd9, 0x26, 0x5a, 0x43, 0x88, 0xd6, 0xcf, 0x32, 0xfb, 0xbd, 0x5f, 0xd8, 0x87,
+ 0x44, 0x0f, 0xa1, 0x44, 0x49, 0xc7, 0xd4, 0x14, 0xea, 0xe5, 0xa0, 0xd7, 0xc2, 0x53, 0x60, 0x91,
+ 0xc3, 0xc0, 0x36, 0x8c, 0xd6, 0xa6, 0x60, 0xe3, 0xdb, 0xc7, 0x77, 0x89, 0x37, 0x8a, 0x7d, 0x18,
+ 0xb4, 0x0f, 0xe3, 0x8e, 0xd9, 0x62, 0x9c, 0x94, 0xd5, 0xb0, 0x76, 0x57, 0x44, 0xd2, 0xad, 0x7e,
+ 0x7d, 0xb3, 0x15, 0x91, 0xae, 0xcf, 0x08, 0x5d, 0xe3, 0xd1, 0x71, 0x1c, 0xd3, 0x82, 0x16, 0x60,
+ 0xa2, 0xa3, 0xea, 0x98, 0x28, 0xad, 0x6e, 0x83, 0x34, 0x0d, 0xbd, 0x65, 0xf3, 0xb0, 0x1a, 0xaa,
+ 0xcf, 0x0a, 0x80, 0x89, 0xd5, 0x28, 0x19, 0xc7, 0xf9, 0xd1, 0xfb, 0x80, 0xbc, 0x69, 0xdc, 0x77,
+ 0x4b, 0xb0, 0x6a, 0xe8, 0x3c, 0xe6, 0xf2, 0x41, 0x70, 0x6f, 0x26, 0x38, 0x70, 0x8a, 0x14, 0x5a,
+ 0x81, 0x73, 0x16, 0xd9, 0x57, 0xd9, 0x1c, 0x1f, 0xa8, 0x36, 0x35, 0xac, 0xee, 0x8a, 0xda, 0x51,
+ 0x69, 0x79, 0x98, 0xdb, 0x54, 0x3e, 0x3a, 0xac, 0x9c, 0xc3, 0x29, 0x74, 0x9c, 0x2a, 0x25, 0xff,
+ 0xcb, 0x30, 0x4c, 0xc4, 0xf2, 0x0d, 0x7a, 0x04, 0x33, 0x4d, 0xb7, 0x38, 0xad, 0x39, 0x9d, 0x6d,
+ 0x62, 0x35, 0x9a, 0xbb, 0xa4, 0xe5, 0x68, 0xa4, 0xc5, 0x03, 0x65, 0xa8, 0x3e, 0x2f, 0x2c, 0x9e,
+ 0x59, 0x4c, 0xe5, 0xc2, 0x19, 0xd2, 0xcc, 0x0b, 0x3a, 0x1f, 0x5a, 0x55, 0x6d, 0xdb, 0xc7, 0xcc,
+ 0x71, 0x4c, 0xdf, 0x0b, 0x6b, 0x09, 0x0e, 0x9c, 0x22, 0xc5, 0x6c, 0x6c, 0x11, 0x5b, 0xb5, 0x48,
+ 0x2b, 0x6e, 0x63, 0x3e, 0x6a, 0xe3, 0x52, 0x2a, 0x17, 0xce, 0x90, 0x46, 0x37, 0x61, 0xd4, 0xd5,
+ 0xc6, 0xd7, 0x4f, 0x2c, 0xb4, 0x5f, 0x0e, 0xd7, 0x02, 0x12, 0x0e, 0xf3, 0xb1, 0xa9, 0x19, 0xdb,
+ 0x36, 0xb1, 0xf6, 0x49, 0x2b, 0x7b, 0x81, 0xd7, 0x13, 0x1c, 0x38, 0x45, 0x8a, 0x4d, 0xcd, 0x8d,
+ 0xc0, 0xc4, 0xd4, 0x86, 0xa3, 0x53, 0xdb, 0x4a, 0xe5, 0xc2, 0x19, 0xd2, 0x2c, 0x8e, 0x5d, 0x93,
+ 0x17, 0xf6, 0x15, 0x55, 0x53, 0xb6, 0x35, 0x52, 0x2e, 0x46, 0xe3, 0x78, 0x2d, 0x4a, 0xc6, 0x71,
+ 0x7e, 0x74, 0x1f, 0xa6, 0xdc, 0xa1, 0x2d, 0x5d, 0xf1, 0x41, 0x4a, 0x1c, 0xe4, 0x65, 0x01, 0x32,
+ 0xb5, 0x16, 0x67, 0xc0, 0x49, 0x19, 0xf4, 0x0e, 0x8c, 0x37, 0x0d, 0x4d, 0xe3, 0xf1, 0xb8, 0x68,
+ 0x38, 0x3a, 0x2d, 0x8f, 0x70, 0x14, 0xc4, 0xf6, 0xe3, 0x62, 0x84, 0x82, 0x63, 0x9c, 0x88, 0x00,
+ 0x34, 0xbd, 0x82, 0x63, 0x97, 0xa1, 0xaf, 0x5e, 0x23, 0x59, 0xf4, 0x82, 0x1e, 0xc0, 0x1f, 0xb2,
+ 0x71, 0x08, 0x58, 0xfe, 0xb1, 0x04, 0xb3, 0x19, 0xa9, 0x03, 0xbd, 0x17, 0x29, 0xb1, 0xbf, 0x1f,
+ 0x2b, 0xb1, 0xe7, 0x33, 0xc4, 0x42, 0x75, 0x56, 0x87, 0x31, 0x8b, 0xcd, 0x4a, 0x6f, 0xbb, 0x2c,
+ 0x22, 0x47, 0xde, 0xec, 0x31, 0x0d, 0x1c, 0x96, 0x09, 0x72, 0xfe, 0xd4, 0xd1, 0x61, 0x65, 0x2c,
+ 0x42, 0xc3, 0x51, 0x78, 0xf9, 0x5f, 0x73, 0x00, 0x4b, 0xc4, 0xd4, 0x8c, 0x6e, 0x87, 0xe8, 0xa7,
+ 0xd1, 0x43, 0xad, 0x47, 0x7a, 0xa8, 0x6b, 0xbd, 0x96, 0xc7, 0x37, 0x2d, 0xb3, 0x89, 0xfa, 0x93,
+ 0x58, 0x13, 0x55, 0xeb, 0x1f, 0xf2, 0xf8, 0x2e, 0xea, 0xa7, 0x79, 0x98, 0x0e, 0x98, 0x83, 0x36,
+ 0xea, 0x4e, 0x64, 0x8d, 0x7f, 0x2f, 0xb6, 0xc6, 0xb3, 0x29, 0x22, 0x2f, 0xac, 0x8f, 0x7a, 0xfe,
+ 0xfd, 0x0c, 0x7a, 0x02, 0xe3, 0xac, 0x71, 0x72, 0xc3, 0x83, 0xb7, 0x65, 0xc3, 0x03, 0xb7, 0x65,
+ 0x7e, 0x01, 0x5d, 0x89, 0x20, 0xe1, 0x18, 0x72, 0x46, 0x1b, 0x58, 0x7c, 0xd1, 0x6d, 0xa0, 0xfc,
+ 0xa5, 0x04, 0xe3, 0xc1, 0x32, 0x9d, 0x42, 0xd3, 0xb6, 0x16, 0x6d, 0xda, 0xae, 0xf4, 0x1d, 0xa2,
+ 0x19, 0x5d, 0xdb, 0x2f, 0x59, 0x83, 0xef, 0x33, 0xb1, 0x0d, 0xbe, 0xad, 0x34, 0xf7, 0xfa, 0xf8,
+ 0xfc, 0xfb, 0x4c, 0x02, 0x24, 0xaa, 0xc0, 0x82, 0xae, 0x1b, 0x54, 0x71, 0x73, 0xa5, 0x6b, 0xd6,
+ 0x72, 0xdf, 0x66, 0x79, 0x1a, 0xab, 0x5b, 0x09, 0xac, 0xbb, 0x3a, 0xb5, 0xba, 0xc1, 0x8a, 0x24,
+ 0x19, 0x70, 0x8a, 0x01, 0x48, 0x01, 0xb0, 0x04, 0xe6, 0xa6, 0x21, 0x36, 0xf2, 0xb5, 0x3e, 0x72,
+ 0x1e, 0x13, 0x58, 0x34, 0xf4, 0x1d, 0xb5, 0x1d, 0xa4, 0x1d, 0xec, 0x03, 0xe1, 0x10, 0xe8, 0xdc,
+ 0x5d, 0x98, 0xcd, 0xb0, 0x16, 0x4d, 0x42, 0x7e, 0x8f, 0x74, 0x5d, 0xb7, 0x61, 0xf6, 0x27, 0x3a,
+ 0x17, 0xfe, 0x4c, 0x1e, 0x11, 0x5f, 0xb8, 0xef, 0xe4, 0x6e, 0x4b, 0xf2, 0x17, 0x43, 0xe1, 0xd8,
+ 0xe1, 0x1d, 0xf3, 0x65, 0x28, 0x59, 0xc4, 0xd4, 0xd4, 0xa6, 0x62, 0x8b, 0x46, 0x88, 0x37, 0xbf,
+ 0x58, 0x8c, 0x61, 0x9f, 0x1a, 0xe9, 0xad, 0x73, 0x2f, 0xb6, 0xb7, 0xce, 0x3f, 0x9f, 0xde, 0xfa,
+ 0xcf, 0xa1, 0x64, 0x7b, 0x5d, 0x75, 0x81, 0x43, 0x5e, 0x1f, 0x20, 0xbf, 0x8a, 0x86, 0xda, 0x57,
+ 0xe0, 0xb7, 0xd2, 0x3e, 0x68, 0x5a, 0x13, 0x3d, 0x34, 0x60, 0x13, 0xfd, 0x5c, 0x1b, 0x5f, 0x96,
+ 0x53, 0x4d, 0xc5, 0xb1, 0x49, 0x8b, 0x27, 0xa2, 0x52, 0x90, 0x53, 0x37, 0xf8, 0x28, 0x16, 0x54,
+ 0xf4, 0x38, 0x12, 0xb2, 0xa5, 0x93, 0x84, 0xec, 0x78, 0x76, 0xb8, 0xa2, 0x2d, 0x98, 0x35, 0x2d,
+ 0xa3, 0x6d, 0x11, 0xdb, 0x5e, 0x22, 0x4a, 0x4b, 0x53, 0x75, 0xe2, 0xf9, 0xc7, 0xed, 0x88, 0xce,
+ 0x1f, 0x1d, 0x56, 0x66, 0x37, 0xd2, 0x59, 0x70, 0x96, 0xac, 0xfc, 0xb4, 0x00, 0x93, 0xf1, 0x0a,
+ 0x98, 0xd1, 0xa4, 0x4a, 0x27, 0x6a, 0x52, 0xaf, 0x86, 0x36, 0x83, 0xdb, 0xc1, 0xfb, 0xab, 0x9f,
+ 0xb2, 0x21, 0x16, 0x60, 0x42, 0x64, 0x03, 0x8f, 0x28, 0xda, 0x74, 0x7f, 0xf5, 0xb7, 0xa2, 0x64,
+ 0x1c, 0xe7, 0x67, 0xad, 0x67, 0xd0, 0x51, 0x7a, 0x20, 0x85, 0x68, 0xeb, 0xb9, 0x10, 0x67, 0xc0,
+ 0x49, 0x19, 0xb4, 0x0a, 0xd3, 0x8e, 0x9e, 0x84, 0x72, 0xa3, 0xf1, 0xbc, 0x80, 0x9a, 0xde, 0x4a,
+ 0xb2, 0xe0, 0x34, 0x39, 0xb4, 0x13, 0xe9, 0x46, 0x87, 0x79, 0x86, 0xbd, 0xd1, 0xf7, 0xde, 0xe9,
+ 0xbb, 0x1d, 0x45, 0x77, 0x60, 0xcc, 0xe2, 0xdf, 0x1d, 0x9e, 0xc1, 0x6e, 0xef, 0xfe, 0x92, 0x10,
+ 0x1b, 0xc3, 0x61, 0x22, 0x8e, 0xf2, 0xa6, 0xb4, 0xdb, 0xa5, 0x7e, 0xdb, 0x6d, 0xf9, 0xff, 0xa5,
+ 0x70, 0x11, 0xf2, 0x5b, 0xe0, 0x5e, 0xa7, 0x4c, 0x09, 0x89, 0x50, 0x77, 0x64, 0xa4, 0x77, 0xbf,
+ 0xb7, 0x06, 0xea, 0x7e, 0x83, 0xe2, 0xd9, 0xbb, 0xfd, 0xfd, 0x5c, 0x82, 0x99, 0x7b, 0x8d, 0xfb,
+ 0x96, 0xe1, 0x98, 0x9e, 0x39, 0xeb, 0xa6, 0xeb, 0xd7, 0xb7, 0xa1, 0x60, 0x39, 0x9a, 0x37, 0x8f,
+ 0xd7, 0xbc, 0x79, 0x60, 0x47, 0x63, 0xf3, 0x98, 0x8e, 0x49, 0xb9, 0x93, 0x60, 0x02, 0x68, 0x0d,
+ 0x86, 0x2d, 0x45, 0x6f, 0x13, 0xaf, 0xac, 0x5e, 0xea, 0x61, 0xfd, 0xf2, 0x12, 0x66, 0xec, 0xa1,
+ 0xe6, 0x8d, 0x4b, 0x63, 0x81, 0x22, 0xff, 0xbd, 0x04, 0x13, 0x0f, 0x36, 0x37, 0x37, 0x96, 0x75,
+ 0xbe, 0xa3, 0xf9, 0x79, 0xfa, 0x45, 0x28, 0x98, 0x0a, 0xdd, 0x8d, 0x57, 0x7a, 0x46, 0xc3, 0x9c,
+ 0x82, 0x3e, 0x80, 0x22, 0xcb, 0x24, 0x44, 0x6f, 0xf5, 0xd9, 0x6a, 0x0b, 0xf8, 0xba, 0x2b, 0x14,
+ 0x74, 0x88, 0x62, 0x00, 0x7b, 0x70, 0xf2, 0x1e, 0x9c, 0x0b, 0x99, 0xc3, 0xfc, 0xc1, 0x8f, 0x81,
+ 0x51, 0x03, 0x86, 0x98, 0x66, 0xef, 0x94, 0xb7, 0xd7, 0x61, 0x66, 0x6c, 0x4a, 0x41, 0xa7, 0xc3,
+ 0x7e, 0xd9, 0xd8, 0xc5, 0x92, 0x57, 0x61, 0x8c, 0x5f, 0x22, 0x18, 0x16, 0xe5, 0x6e, 0x41, 0x17,
+ 0x20, 0xdf, 0x51, 0x75, 0x51, 0x67, 0x47, 0x85, 0x4c, 0x9e, 0xd5, 0x08, 0x36, 0xce, 0xc9, 0xca,
+ 0x81, 0xc8, 0x3c, 0x01, 0x59, 0x39, 0xc0, 0x6c, 0x5c, 0xbe, 0x0f, 0x45, 0xe1, 0xee, 0x30, 0x50,
+ 0xfe, 0x78, 0xa0, 0x7c, 0x0a, 0xd0, 0x3a, 0x14, 0x97, 0x37, 0xea, 0x9a, 0xe1, 0x76, 0x5d, 0x4d,
+ 0xb5, 0x65, 0xc5, 0xd7, 0x62, 0x71, 0x79, 0x09, 0x63, 0x4e, 0x41, 0x32, 0x0c, 0x93, 0x83, 0x26,
+ 0x31, 0x29, 0x8f, 0x88, 0x91, 0x3a, 0xb0, 0x55, 0xbe, 0xcb, 0x47, 0xb0, 0xa0, 0xc8, 0xff, 0x90,
+ 0x83, 0xa2, 0x70, 0xc7, 0x29, 0x7c, 0x85, 0xad, 0x44, 0xbe, 0xc2, 0x5e, 0xef, 0x2f, 0x34, 0x32,
+ 0x3f, 0xc1, 0x36, 0x63, 0x9f, 0x60, 0x57, 0xfb, 0xc4, 0x3b, 0xfe, 0xfb, 0xeb, 0x7f, 0x24, 0x18,
+ 0x8f, 0x06, 0x25, 0xba, 0x09, 0xa3, 0xac, 0xe0, 0xa8, 0x4d, 0xb2, 0x16, 0xf4, 0xb9, 0xfe, 0x21,
+ 0x4c, 0x23, 0x20, 0xe1, 0x30, 0x1f, 0x6a, 0xfb, 0x62, 0x2c, 0x8e, 0xc4, 0xa4, 0xb3, 0x5d, 0xea,
+ 0x50, 0x55, 0xab, 0xba, 0x17, 0x63, 0xd5, 0x65, 0x9d, 0xae, 0x5b, 0x0d, 0x6a, 0xa9, 0x7a, 0x3b,
+ 0xa1, 0x88, 0x07, 0x65, 0x18, 0x59, 0xfe, 0x3f, 0x09, 0x46, 0x85, 0xc9, 0xa7, 0xf0, 0x55, 0xf1,
+ 0xc7, 0xd1, 0xaf, 0x8a, 0x4b, 0x7d, 0x6e, 0xf0, 0xf4, 0x4f, 0x8a, 0xff, 0x08, 0x4c, 0x67, 0x5b,
+ 0x9a, 0x45, 0xf5, 0xae, 0x61, 0xd3, 0x78, 0x54, 0xb3, 0xcd, 0x88, 0x39, 0x05, 0x39, 0x30, 0xa9,
+ 0xc6, 0x72, 0x80, 0x70, 0x6d, 0xad, 0x3f, 0x4b, 0x7c, 0xb1, 0x7a, 0x59, 0xc0, 0x4f, 0xc6, 0x29,
+ 0x38, 0xa1, 0x42, 0x26, 0x90, 0xe0, 0x42, 0x0f, 0xa1, 0xb0, 0x4b, 0xa9, 0x99, 0x72, 0x5e, 0xdd,
+ 0x23, 0xf3, 0x04, 0x26, 0x94, 0xf8, 0xec, 0x36, 0x37, 0x37, 0x30, 0x87, 0x92, 0x7f, 0x15, 0xf8,
+ 0xa3, 0xe1, 0xc6, 0xb8, 0x9f, 0x4f, 0xa5, 0x93, 0xe4, 0xd3, 0xd1, 0xb4, 0x5c, 0x8a, 0x1e, 0x40,
+ 0x9e, 0x6a, 0xfd, 0x7e, 0x16, 0x0a, 0xc4, 0xcd, 0x95, 0x46, 0x90, 0x90, 0x36, 0x57, 0x1a, 0x98,
+ 0x41, 0xa0, 0x75, 0x18, 0x62, 0xd5, 0x87, 0x6d, 0xc1, 0x7c, 0xff, 0x5b, 0x9a, 0xcd, 0x3f, 0x08,
+ 0x08, 0xf6, 0xcb, 0xc6, 0x2e, 0x8e, 0xfc, 0x09, 0x8c, 0x45, 0xf6, 0x29, 0xfa, 0x18, 0xce, 0x6a,
+ 0x86, 0xd2, 0xaa, 0x2b, 0x9a, 0xa2, 0x37, 0x89, 0x77, 0x39, 0x70, 0x29, 0xed, 0x0b, 0x63, 0x25,
+ 0xc4, 0x27, 0x76, 0xb9, 0x7f, 0x9d, 0x1a, 0xa6, 0xe1, 0x08, 0xa2, 0xac, 0x00, 0x04, 0x73, 0x44,
+ 0x15, 0x18, 0x62, 0x71, 0xe6, 0xd6, 0x93, 0x91, 0xfa, 0x08, 0xb3, 0x90, 0x85, 0x9f, 0x8d, 0xdd,
+ 0x71, 0x74, 0x03, 0xc0, 0x26, 0x4d, 0x8b, 0x50, 0x9e, 0x0c, 0x72, 0xd1, 0x4b, 0xe5, 0x86, 0x4f,
+ 0xc1, 0x21, 0x2e, 0xf9, 0x47, 0x12, 0x8c, 0xad, 0x11, 0xfa, 0xa9, 0x61, 0xed, 0x6d, 0xf0, 0xc7,
+ 0x00, 0xa7, 0x90, 0x6c, 0x71, 0x24, 0xd9, 0xbe, 0xd1, 0x63, 0x65, 0x22, 0xd6, 0x65, 0xa5, 0x5c,
+ 0xf9, 0x4b, 0x09, 0x66, 0x23, 0x9c, 0x77, 0x83, 0xad, 0xbb, 0x05, 0x43, 0xa6, 0x61, 0x51, 0xaf,
+ 0x10, 0x0f, 0xa4, 0x90, 0xa5, 0xb1, 0x50, 0x29, 0x66, 0x30, 0xd8, 0x45, 0x43, 0x2b, 0x90, 0xa3,
+ 0x86, 0x08, 0xd5, 0xc1, 0x30, 0x09, 0xb1, 0xea, 0x20, 0x30, 0x73, 0x9b, 0x06, 0xce, 0x51, 0x83,
+ 0x2d, 0x44, 0x39, 0xc2, 0x15, 0x4e, 0x3e, 0x2f, 0x68, 0x06, 0x18, 0x0a, 0x3b, 0x96, 0xd1, 0x39,
+ 0xf1, 0x1c, 0xfc, 0x85, 0xb8, 0x67, 0x19, 0x1d, 0xcc, 0xb1, 0xe4, 0xaf, 0x24, 0x98, 0x8a, 0x70,
+ 0x9e, 0x42, 0xe2, 0x7f, 0x18, 0x4d, 0xfc, 0x57, 0x07, 0x99, 0x48, 0x46, 0xfa, 0xff, 0x2a, 0x17,
+ 0x9b, 0x06, 0x9b, 0x30, 0xda, 0x81, 0x51, 0xd3, 0x68, 0x35, 0x9e, 0xc3, 0x75, 0xe0, 0x04, 0xab,
+ 0x9b, 0x1b, 0x01, 0x16, 0x0e, 0x03, 0xa3, 0x03, 0x98, 0xd2, 0x95, 0x0e, 0xb1, 0x4d, 0xa5, 0x49,
+ 0x1a, 0xcf, 0xe1, 0x80, 0xe4, 0x25, 0x7e, 0xdf, 0x10, 0x47, 0xc4, 0x49, 0x25, 0x68, 0x15, 0x8a,
+ 0xaa, 0xc9, 0xfb, 0x38, 0xd1, 0xbb, 0xf4, 0xac, 0xa2, 0x6e, 0xd7, 0xe7, 0xe6, 0x73, 0xf1, 0x03,
+ 0x7b, 0x18, 0xf2, 0x7f, 0xc6, 0xa3, 0x81, 0xc5, 0x1f, 0xba, 0x0f, 0x25, 0xfe, 0xac, 0xa6, 0x69,
+ 0x68, 0xde, 0xcd, 0x00, 0x5b, 0xd9, 0x0d, 0x31, 0xf6, 0xec, 0xb0, 0x72, 0x3e, 0xe5, 0xd0, 0xd7,
+ 0x23, 0x63, 0x5f, 0x18, 0xad, 0x41, 0xc1, 0xfc, 0x21, 0x1d, 0x0c, 0x2f, 0x72, 0xbc, 0x6d, 0xe1,
+ 0x38, 0xf2, 0x5f, 0xe7, 0x63, 0xe6, 0xf2, 0x52, 0xf7, 0xe4, 0xb9, 0xad, 0xba, 0xdf, 0x31, 0x65,
+ 0xae, 0xfc, 0x36, 0x14, 0x45, 0x85, 0x17, 0xc1, 0xfc, 0xf6, 0x20, 0xc1, 0x1c, 0xae, 0x62, 0xfe,
+ 0x07, 0x8b, 0x37, 0xe8, 0x01, 0xa3, 0x8f, 0x60, 0x98, 0xb8, 0x2a, 0xdc, 0xda, 0x78, 0x6b, 0x10,
+ 0x15, 0x41, 0x5e, 0x0d, 0x1a, 0x55, 0x31, 0x26, 0x50, 0xd1, 0x7b, 0xcc, 0x5f, 0x8c, 0x97, 0x7d,
+ 0x04, 0xda, 0xe5, 0x02, 0x2f, 0x57, 0x17, 0xdc, 0x69, 0xfb, 0xc3, 0xcf, 0x0e, 0x2b, 0x10, 0xfc,
+ 0xc4, 0x61, 0x09, 0xf9, 0x27, 0x12, 0x4c, 0x71, 0x0f, 0x35, 0x1d, 0x4b, 0xa5, 0xdd, 0x53, 0x2b,
+ 0x4c, 0x8f, 0x22, 0x85, 0xe9, 0xad, 0x1e, 0x6e, 0x49, 0x58, 0x98, 0x59, 0x9c, 0xbe, 0x96, 0xe0,
+ 0xa5, 0x04, 0xf7, 0x29, 0xe4, 0xc5, 0xad, 0x68, 0x5e, 0x7c, 0x63, 0xd0, 0x09, 0x65, 0xbd, 0x91,
+ 0x18, 0x4b, 0x99, 0x0e, 0xdf, 0x29, 0x37, 0x00, 0x4c, 0x4b, 0xdd, 0x57, 0x35, 0xd2, 0x16, 0x97,
+ 0xe0, 0xa5, 0xd0, 0xb3, 0x36, 0x9f, 0x82, 0x43, 0x5c, 0xc8, 0x86, 0x99, 0x16, 0xd9, 0x51, 0x1c,
+ 0x8d, 0x2e, 0xb4, 0x5a, 0x8b, 0x8a, 0xa9, 0x6c, 0xab, 0x9a, 0x4a, 0x55, 0x71, 0x5c, 0x30, 0x52,
+ 0xbf, 0xe3, 0x5e, 0x4e, 0xa7, 0x71, 0x3c, 0x3b, 0xac, 0x5c, 0x48, 0xbb, 0x1d, 0xf2, 0x58, 0xba,
+ 0x38, 0x03, 0x1a, 0x75, 0xa1, 0x6c, 0x91, 0x4f, 0x1c, 0xd5, 0x22, 0xad, 0x25, 0xcb, 0x30, 0x23,
+ 0x6a, 0xf3, 0x5c, 0xed, 0x1f, 0x1e, 0x1d, 0x56, 0xca, 0x38, 0x83, 0xa7, 0xb7, 0xe2, 0x4c, 0x78,
+ 0xf4, 0x04, 0xa6, 0x15, 0xf7, 0x35, 0x60, 0x44, 0xab, 0xbb, 0x4b, 0x6e, 0x1f, 0x1d, 0x56, 0xa6,
+ 0x17, 0x92, 0xe4, 0xde, 0x0a, 0xd3, 0x40, 0x51, 0x0d, 0x8a, 0xfb, 0xfc, 0xad, 0xa2, 0x5d, 0x1e,
+ 0xe2, 0xf8, 0xac, 0x10, 0x14, 0xdd, 0xe7, 0x8b, 0x0c, 0x73, 0xf8, 0x5e, 0x83, 0xef, 0x3e, 0x8f,
+ 0x8b, 0x7d, 0x50, 0xb2, 0x5e, 0x52, 0xec, 0x78, 0x7e, 0x62, 0x5c, 0x0a, 0xb2, 0xd6, 0x83, 0x80,
+ 0x84, 0xc3, 0x7c, 0xe8, 0x31, 0x8c, 0xec, 0x8a, 0x53, 0x09, 0xbb, 0x5c, 0xec, 0xab, 0x08, 0x47,
+ 0x4e, 0x31, 0xea, 0x53, 0x42, 0xc5, 0x88, 0x37, 0x6c, 0xe3, 0x00, 0x11, 0x5d, 0x81, 0x22, 0xff,
+ 0xb1, 0xbc, 0xc4, 0x8f, 0xe3, 0x4a, 0x41, 0x6e, 0x7b, 0xe0, 0x0e, 0x63, 0x8f, 0xee, 0xb1, 0x2e,
+ 0x6f, 0x2c, 0xf2, 0x63, 0xe1, 0x18, 0xeb, 0xf2, 0xc6, 0x22, 0xf6, 0xe8, 0xe8, 0x63, 0x28, 0xda,
+ 0x64, 0x45, 0xd5, 0x9d, 0x83, 0x32, 0xf4, 0x75, 0xa9, 0xdc, 0xb8, 0xcb, 0xb9, 0x63, 0x07, 0x63,
+ 0x81, 0x06, 0x41, 0xc7, 0x1e, 0x2c, 0xda, 0x85, 0x11, 0xcb, 0xd1, 0x17, 0xec, 0x2d, 0x9b, 0x58,
+ 0xe5, 0x51, 0xae, 0xa3, 0x57, 0x3a, 0xc7, 0x1e, 0x7f, 0x5c, 0x8b, 0xef, 0x21, 0x9f, 0x03, 0x07,
+ 0xe0, 0xe8, 0xef, 0x24, 0x40, 0xb6, 0x63, 0x9a, 0x1a, 0xe9, 0x10, 0x9d, 0x2a, 0x1a, 0x3f, 0x8b,
+ 0xb3, 0xcb, 0x67, 0xb9, 0xce, 0x3f, 0xea, 0x35, 0xaf, 0x84, 0x60, 0x5c, 0xb9, 0x7f, 0xe8, 0x9d,
+ 0x64, 0xc5, 0x29, 0x7a, 0x99, 0x6b, 0x77, 0x6c, 0xfe, 0x77, 0x79, 0xac, 0x2f, 0xd7, 0xa6, 0x9f,
+ 0x39, 0x06, 0xae, 0x15, 0x74, 0xec, 0xc1, 0xa2, 0x47, 0x30, 0x63, 0x11, 0xa5, 0xb5, 0xae, 0x6b,
+ 0x5d, 0x6c, 0x18, 0xf4, 0x9e, 0xaa, 0x11, 0xbb, 0x6b, 0x53, 0xd2, 0x29, 0x8f, 0xf3, 0x65, 0xf7,
+ 0xdf, 0x7e, 0xe0, 0x54, 0x2e, 0x9c, 0x21, 0x8d, 0x3a, 0x50, 0xf1, 0x52, 0x06, 0xdb, 0x4f, 0x7e,
+ 0xce, 0xba, 0x6b, 0x37, 0x15, 0xcd, 0xbd, 0x07, 0x98, 0xe0, 0x0a, 0x5e, 0x3b, 0x3a, 0xac, 0x54,
+ 0x96, 0x8e, 0x67, 0xc5, 0xbd, 0xb0, 0xd0, 0x07, 0x50, 0x56, 0xb2, 0xf4, 0x4c, 0x72, 0x3d, 0xaf,
+ 0xb0, 0x3c, 0x94, 0xa9, 0x20, 0x53, 0x1a, 0x51, 0x98, 0x54, 0xa2, 0x8f, 0x8e, 0xed, 0xf2, 0x54,
+ 0x5f, 0x07, 0x91, 0xb1, 0xb7, 0xca, 0xc1, 0x61, 0x44, 0x8c, 0x60, 0xe3, 0x84, 0x06, 0xf4, 0x17,
+ 0x80, 0x94, 0xf8, 0x3b, 0x69, 0xbb, 0x8c, 0xfa, 0x2a, 0x3f, 0x89, 0x07, 0xd6, 0x41, 0xd8, 0x25,
+ 0x48, 0x36, 0x4e, 0xd1, 0xc3, 0x1f, 0x6f, 0x88, 0xa3, 0xfc, 0xd3, 0x79, 0x00, 0x3b, 0xd8, 0xe3,
+ 0x8d, 0xc0, 0xb4, 0xe7, 0xf6, 0x78, 0x23, 0x04, 0x79, 0xfc, 0xe1, 0xe1, 0x2f, 0x72, 0x30, 0x1d,
+ 0x30, 0xf7, 0xfd, 0x78, 0x23, 0x45, 0xe4, 0x77, 0x8f, 0x60, 0x7b, 0x3f, 0x82, 0xfd, 0x52, 0x82,
+ 0xf1, 0xc0, 0x75, 0xbf, 0x79, 0x0f, 0x2a, 0x02, 0xdb, 0x32, 0x5a, 0xbc, 0xff, 0xce, 0x85, 0x27,
+ 0xf0, 0x5b, 0x7f, 0xab, 0xff, 0xc3, 0x5f, 0xae, 0xca, 0x5f, 0xe7, 0x61, 0x32, 0xbe, 0x1b, 0x23,
+ 0x97, 0xbf, 0x52, 0xcf, 0xcb, 0xdf, 0x0d, 0x38, 0xb7, 0xe3, 0x68, 0x5a, 0x97, 0xbb, 0x21, 0x74,
+ 0x03, 0xec, 0x5e, 0xde, 0xbc, 0x22, 0x24, 0xcf, 0xdd, 0x4b, 0xe1, 0xc1, 0xa9, 0x92, 0x19, 0x17,
+ 0xd9, 0xf9, 0x13, 0x5d, 0x64, 0x27, 0xee, 0x55, 0x0b, 0x03, 0xdc, 0xab, 0xa6, 0x5e, 0x4a, 0x0f,
+ 0x9d, 0xe0, 0x52, 0xfa, 0x24, 0xb7, 0xc8, 0x29, 0x49, 0xac, 0xe7, 0xa3, 0xc6, 0x57, 0x60, 0x4e,
+ 0x88, 0x51, 0x7e, 0xc1, 0xab, 0x53, 0xcb, 0xd0, 0x34, 0x62, 0x2d, 0x39, 0x9d, 0x4e, 0x57, 0x7e,
+ 0x17, 0xc6, 0xa3, 0x4f, 0x17, 0xdc, 0x95, 0x76, 0x5f, 0x4f, 0x88, 0x2b, 0xb4, 0xd0, 0x4a, 0xbb,
+ 0xe3, 0xd8, 0xe7, 0x90, 0xff, 0x46, 0x82, 0x99, 0xf4, 0x27, 0x8a, 0x48, 0x83, 0xf1, 0x8e, 0x72,
+ 0x10, 0x7e, 0x36, 0x2a, 0x9d, 0xf0, 0x70, 0x83, 0xdf, 0x59, 0xaf, 0x46, 0xb0, 0x70, 0x0c, 0x5b,
+ 0xfe, 0x5e, 0x82, 0xd9, 0x8c, 0xdb, 0xe2, 0xd3, 0xb5, 0x04, 0x7d, 0x08, 0xa5, 0x8e, 0x72, 0xd0,
+ 0x70, 0xac, 0x36, 0x39, 0xf1, 0x71, 0x0e, 0xcf, 0x18, 0xab, 0x02, 0x05, 0xfb, 0x78, 0xf2, 0xe7,
+ 0x12, 0x94, 0xb3, 0x1a, 0x6b, 0x74, 0x33, 0x72, 0xaf, 0xfd, 0x6a, 0xec, 0x5e, 0x7b, 0x2a, 0x21,
+ 0xf7, 0x82, 0x6e, 0xb5, 0xff, 0x4b, 0x82, 0x99, 0xf4, 0x0f, 0x0c, 0xf4, 0x66, 0xc4, 0xc2, 0x4a,
+ 0xcc, 0xc2, 0x89, 0x98, 0x94, 0xb0, 0xef, 0x23, 0x18, 0x17, 0x9f, 0x21, 0x02, 0x46, 0x78, 0x55,
+ 0x4e, 0xcb, 0x95, 0x02, 0xc2, 0x6b, 0xbb, 0xf9, 0x7a, 0x45, 0xc7, 0x70, 0x0c, 0x4d, 0xfe, 0xdb,
+ 0x1c, 0x0c, 0x35, 0x9a, 0x8a, 0x46, 0x4e, 0xa1, 0xcd, 0x7a, 0x3f, 0xd2, 0x66, 0xf5, 0xfa, 0x17,
+ 0x0f, 0x6e, 0x55, 0x66, 0x87, 0x85, 0x63, 0x1d, 0xd6, 0xeb, 0x7d, 0xa1, 0x1d, 0xdf, 0x5c, 0xfd,
+ 0x01, 0x8c, 0xf8, 0x4a, 0x07, 0xcb, 0xf9, 0xf2, 0xbf, 0xe7, 0x60, 0x34, 0xa4, 0x62, 0xc0, 0x8a,
+ 0xb1, 0x13, 0xa9, 0xb4, 0xfd, 0xfc, 0x63, 0x5d, 0x48, 0x57, 0xd5, 0xab, 0xad, 0xee, 0x13, 0xc5,
+ 0xe0, 0x51, 0x5a, 0xb2, 0xe4, 0xbe, 0x0b, 0xe3, 0x94, 0xff, 0xe3, 0x99, 0x7f, 0x08, 0x9a, 0xe7,
+ 0xb1, 0xe8, 0x3f, 0x6c, 0xdd, 0x8c, 0x50, 0x71, 0x8c, 0x7b, 0xee, 0x0e, 0x8c, 0x45, 0x94, 0x0d,
+ 0xf4, 0xc2, 0xf0, 0x7f, 0x25, 0x78, 0xb5, 0xe7, 0x27, 0x2a, 0xaa, 0x47, 0x36, 0x49, 0x35, 0xb6,
+ 0x49, 0xe6, 0xb3, 0x01, 0x5e, 0xdc, 0x4b, 0x95, 0xfa, 0xb5, 0xa7, 0xdf, 0xcd, 0x9f, 0xf9, 0xe6,
+ 0xbb, 0xf9, 0x33, 0xdf, 0x7e, 0x37, 0x7f, 0xe6, 0xaf, 0x8e, 0xe6, 0xa5, 0xa7, 0x47, 0xf3, 0xd2,
+ 0x37, 0x47, 0xf3, 0xd2, 0xb7, 0x47, 0xf3, 0xd2, 0xcf, 0x8e, 0xe6, 0xa5, 0x7f, 0xfc, 0x7e, 0xfe,
+ 0xcc, 0x87, 0x45, 0x01, 0xf7, 0xeb, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0xb3, 0xc8, 0xe2, 0x54,
+ 0x3c, 0x00, 0x00,
}
diff --git a/staging/src/k8s.io/api/extensions/v1beta1/generated.proto b/staging/src/k8s.io/api/extensions/v1beta1/generated.proto
index f6898bce640..99d034373dd 100644
--- a/staging/src/k8s.io/api/extensions/v1beta1/generated.proto
+++ b/staging/src/k8s.io/api/extensions/v1beta1/generated.proto
@@ -32,6 +32,12 @@ import "k8s.io/apimachinery/pkg/util/intstr/generated.proto";
// Package-wide variables from generator "generated".
option go_package = "v1beta1";
+// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
+message AllowedFlexVolume {
+ // Driver is the name of the Flexvolume driver.
+ optional string driver = 1;
+}
+
// defines the host volume conditions that will be enabled by a policy
// for pods to use. It requires the path prefix to be defined.
message AllowedHostPath {
@@ -843,6 +849,12 @@ message PodSecurityPolicySpec {
// is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
repeated AllowedHostPath allowedHostPaths = 17;
+
+ // AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
+ // Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
+ // is allowed in the "Volumes" field.
+ // +optional
+ repeated AllowedFlexVolume allowedFlexVolumes = 18;
}
// DEPRECATED - This group version of ReplicaSet is deprecated by apps/v1beta2/ReplicaSet. See the release notes for
diff --git a/staging/src/k8s.io/api/extensions/v1beta1/types.go b/staging/src/k8s.io/api/extensions/v1beta1/types.go
index d4ca1832ca5..9be564ed88e 100644
--- a/staging/src/k8s.io/api/extensions/v1beta1/types.go
+++ b/staging/src/k8s.io/api/extensions/v1beta1/types.go
@@ -938,6 +938,11 @@ type PodSecurityPolicySpec struct {
// is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
AllowedHostPaths []AllowedHostPath `json:"allowedHostPaths,omitempty" protobuf:"bytes,17,rep,name=allowedHostPaths"`
+ // AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
+ // Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
+ // is allowed in the "Volumes" field.
+ // +optional
+ AllowedFlexVolumes []AllowedFlexVolume `json:"allowedFlexVolumes,omitempty" protobuf:"bytes,18,rep,name=allowedFlexVolumes"`
}
// defines the host volume conditions that will be enabled by a policy
@@ -981,6 +986,12 @@ var (
All FSType = "*"
)
+// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
+type AllowedFlexVolume struct {
+ // Driver is the name of the Flexvolume driver.
+ Driver string `json:"driver" protobuf:"bytes,1,opt,name=driver"`
+}
+
// Host Port Range defines a range of host ports that will be enabled by a policy
// for pods to use. It requires both the start and end to be defined.
type HostPortRange struct {
diff --git a/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go
index b98d1179522..c9eb79477cf 100644
--- a/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go
+++ b/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go
@@ -27,6 +27,15 @@ package v1beta1
// Those methods can be generated by using hack/update-generated-swagger-docs.sh
// AUTO-GENERATED FUNCTIONS START HERE
+var map_AllowedFlexVolume = map[string]string{
+ "": "AllowedFlexVolume represents a single Flexvolume that is allowed to be used.",
+ "driver": "Driver is the name of the Flexvolume driver.",
+}
+
+func (AllowedFlexVolume) SwaggerDoc() map[string]string {
+ return map_AllowedFlexVolume
+}
+
var map_AllowedHostPath = map[string]string{
"": "defines the host volume conditions that will be enabled by a policy for pods to use. It requires the path prefix to be defined.",
"pathPrefix": "is the path prefix that the host volume must match. It does not support `*`. Trailing slashes are trimmed when validating the path prefix with a host path.\n\nExamples: `/foo` would allow `/foo`, `/foo/` and `/foo/bar` `/foo` would not allow `/food` or `/etc/foo`",
@@ -465,6 +474,7 @@ var map_PodSecurityPolicySpec = map[string]string{
"defaultAllowPrivilegeEscalation": "DefaultAllowPrivilegeEscalation controls the default setting for whether a process can gain more privileges than its parent process.",
"allowPrivilegeEscalation": "AllowPrivilegeEscalation determines if a pod can request to allow privilege escalation. If unspecified, defaults to true.",
"allowedHostPaths": "is a white list of allowed host paths. Empty indicates that all host paths may be used.",
+ "allowedFlexVolumes": "AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the \"Volumes\" field.",
}
func (PodSecurityPolicySpec) SwaggerDoc() map[string]string {
diff --git a/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go
index 0cc661defd4..02a84cccd2c 100644
--- a/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go
+++ b/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go
@@ -27,6 +27,22 @@ import (
intstr "k8s.io/apimachinery/pkg/util/intstr"
)
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AllowedFlexVolume) DeepCopyInto(out *AllowedFlexVolume) {
+ *out = *in
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedFlexVolume.
+func (in *AllowedFlexVolume) DeepCopy() *AllowedFlexVolume {
+ if in == nil {
+ return nil
+ }
+ out := new(AllowedFlexVolume)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedHostPath) DeepCopyInto(out *AllowedHostPath) {
*out = *in
@@ -1194,6 +1210,11 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
*out = make([]AllowedHostPath, len(*in))
copy(*out, *in)
}
+ if in.AllowedFlexVolumes != nil {
+ in, out := &in.AllowedFlexVolumes, &out.AllowedFlexVolumes
+ *out = make([]AllowedFlexVolume, len(*in))
+ copy(*out, *in)
+ }
return
}
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/BUILD b/staging/src/k8s.io/client-go/listers/apps/v1/BUILD
index 98d6dfe45c3..fb3cfbbb6af 100644
--- a/staging/src/k8s.io/client-go/listers/apps/v1/BUILD
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/BUILD
@@ -5,16 +5,22 @@ go_library(
srcs = [
"controllerrevision.go",
"daemonset.go",
+ "daemonset_expansion.go",
"deployment.go",
+ "deployment_expansion.go",
"expansion_generated.go",
"replicaset.go",
+ "replicaset_expansion.go",
"statefulset.go",
+ "statefulset_expansion.go",
],
importpath = "k8s.io/client-go/listers/apps/v1",
visibility = ["//visibility:public"],
deps = [
"//vendor/k8s.io/api/apps/v1:go_default_library",
+ "//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+ "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/daemonset_expansion.go b/staging/src/k8s.io/client-go/listers/apps/v1/daemonset_expansion.go
new file mode 100644
index 00000000000..83435561a14
--- /dev/null
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/daemonset_expansion.go
@@ -0,0 +1,113 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ "fmt"
+
+ apps "k8s.io/api/apps/v1"
+ "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+)
+
+// DaemonSetListerExpansion allows custom methods to be added to
+// DaemonSetLister.
+type DaemonSetListerExpansion interface {
+ GetPodDaemonSets(pod *v1.Pod) ([]*apps.DaemonSet, error)
+ GetHistoryDaemonSets(history *apps.ControllerRevision) ([]*apps.DaemonSet, error)
+}
+
+// DaemonSetNamespaceListerExpansion allows custom methods to be added to
+// DaemonSetNamespaceLister.
+type DaemonSetNamespaceListerExpansion interface{}
+
+// GetPodDaemonSets returns a list of DaemonSets that potentially match a pod.
+// Only the one specified in the Pod's ControllerRef will actually manage it.
+// Returns an error only if no matching DaemonSets are found.
+func (s *daemonSetLister) GetPodDaemonSets(pod *v1.Pod) ([]*apps.DaemonSet, error) {
+ var selector labels.Selector
+ var daemonSet *apps.DaemonSet
+
+ if len(pod.Labels) == 0 {
+ return nil, fmt.Errorf("no daemon sets found for pod %v because it has no labels", pod.Name)
+ }
+
+ list, err := s.DaemonSets(pod.Namespace).List(labels.Everything())
+ if err != nil {
+ return nil, err
+ }
+
+ var daemonSets []*apps.DaemonSet
+ for i := range list {
+ daemonSet = list[i]
+ if daemonSet.Namespace != pod.Namespace {
+ continue
+ }
+ selector, err = metav1.LabelSelectorAsSelector(daemonSet.Spec.Selector)
+ if err != nil {
+ // this should not happen if the DaemonSet passed validation
+ return nil, err
+ }
+
+ // If a daemonSet with a nil or empty selector creeps in, it should match nothing, not everything.
+ if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
+ continue
+ }
+ daemonSets = append(daemonSets, daemonSet)
+ }
+
+ if len(daemonSets) == 0 {
+ return nil, fmt.Errorf("could not find daemon set for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
+ }
+
+ return daemonSets, nil
+}
+
+// GetHistoryDaemonSets returns a list of DaemonSets that potentially
+// match a ControllerRevision. Only the one specified in the ControllerRevision's ControllerRef
+// will actually manage it.
+// Returns an error only if no matching DaemonSets are found.
+func (s *daemonSetLister) GetHistoryDaemonSets(history *apps.ControllerRevision) ([]*apps.DaemonSet, error) {
+ if len(history.Labels) == 0 {
+ return nil, fmt.Errorf("no DaemonSet found for ControllerRevision %s because it has no labels", history.Name)
+ }
+
+ list, err := s.DaemonSets(history.Namespace).List(labels.Everything())
+ if err != nil {
+ return nil, err
+ }
+
+ var daemonSets []*apps.DaemonSet
+ for _, ds := range list {
+ selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector)
+ if err != nil {
+ return nil, fmt.Errorf("invalid label selector: %v", err)
+ }
+ // If a DaemonSet with a nil or empty selector creeps in, it should match nothing, not everything.
+ if selector.Empty() || !selector.Matches(labels.Set(history.Labels)) {
+ continue
+ }
+ daemonSets = append(daemonSets, ds)
+ }
+
+ if len(daemonSets) == 0 {
+ return nil, fmt.Errorf("could not find DaemonSets for ControllerRevision %s in namespace %s with labels: %v", history.Name, history.Namespace, history.Labels)
+ }
+
+ return daemonSets, nil
+}
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/deployment_expansion.go b/staging/src/k8s.io/client-go/listers/apps/v1/deployment_expansion.go
new file mode 100644
index 00000000000..7802eca5a5e
--- /dev/null
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/deployment_expansion.go
@@ -0,0 +1,70 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ "fmt"
+
+ apps "k8s.io/api/apps/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+)
+
+// DeploymentListerExpansion allows custom methods to be added to
+// DeploymentLister.
+type DeploymentListerExpansion interface {
+ GetDeploymentsForReplicaSet(rs *apps.ReplicaSet) ([]*apps.Deployment, error)
+}
+
+// DeploymentNamespaceListerExpansion allows custom methods to be added to
+// DeploymentNamespaceLister.
+type DeploymentNamespaceListerExpansion interface{}
+
+// GetDeploymentsForReplicaSet returns a list of Deployments that potentially
+// match a ReplicaSet. Only the one specified in the ReplicaSet's ControllerRef
+// will actually manage it.
+// Returns an error only if no matching Deployments are found.
+func (s *deploymentLister) GetDeploymentsForReplicaSet(rs *apps.ReplicaSet) ([]*apps.Deployment, error) {
+ if len(rs.Labels) == 0 {
+ return nil, fmt.Errorf("no deployments found for ReplicaSet %v because it has no labels", rs.Name)
+ }
+
+ // TODO: MODIFY THIS METHOD so that it checks for the podTemplateSpecHash label
+ dList, err := s.Deployments(rs.Namespace).List(labels.Everything())
+ if err != nil {
+ return nil, err
+ }
+
+ var deployments []*apps.Deployment
+ for _, d := range dList {
+ selector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector)
+ if err != nil {
+ return nil, fmt.Errorf("invalid label selector: %v", err)
+ }
+ // If a deployment with a nil or empty selector creeps in, it should match nothing, not everything.
+ if selector.Empty() || !selector.Matches(labels.Set(rs.Labels)) {
+ continue
+ }
+ deployments = append(deployments, d)
+ }
+
+ if len(deployments) == 0 {
+ return nil, fmt.Errorf("could not find deployments set for ReplicaSet %s in namespace %s with labels: %v", rs.Name, rs.Namespace, rs.Labels)
+ }
+
+ return deployments, nil
+}
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/expansion_generated.go b/staging/src/k8s.io/client-go/listers/apps/v1/expansion_generated.go
index 1f17ab48b6a..48917c2c051 100644
--- a/staging/src/k8s.io/client-go/listers/apps/v1/expansion_generated.go
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/expansion_generated.go
@@ -25,35 +25,3 @@ type ControllerRevisionListerExpansion interface{}
// ControllerRevisionNamespaceListerExpansion allows custom methods to be added to
// ControllerRevisionNamespaceLister.
type ControllerRevisionNamespaceListerExpansion interface{}
-
-// DaemonSetListerExpansion allows custom methods to be added to
-// DaemonSetLister.
-type DaemonSetListerExpansion interface{}
-
-// DaemonSetNamespaceListerExpansion allows custom methods to be added to
-// DaemonSetNamespaceLister.
-type DaemonSetNamespaceListerExpansion interface{}
-
-// DeploymentListerExpansion allows custom methods to be added to
-// DeploymentLister.
-type DeploymentListerExpansion interface{}
-
-// DeploymentNamespaceListerExpansion allows custom methods to be added to
-// DeploymentNamespaceLister.
-type DeploymentNamespaceListerExpansion interface{}
-
-// ReplicaSetListerExpansion allows custom methods to be added to
-// ReplicaSetLister.
-type ReplicaSetListerExpansion interface{}
-
-// ReplicaSetNamespaceListerExpansion allows custom methods to be added to
-// ReplicaSetNamespaceLister.
-type ReplicaSetNamespaceListerExpansion interface{}
-
-// StatefulSetListerExpansion allows custom methods to be added to
-// StatefulSetLister.
-type StatefulSetListerExpansion interface{}
-
-// StatefulSetNamespaceListerExpansion allows custom methods to be added to
-// StatefulSetNamespaceLister.
-type StatefulSetNamespaceListerExpansion interface{}
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/replicaset_expansion.go b/staging/src/k8s.io/client-go/listers/apps/v1/replicaset_expansion.go
new file mode 100644
index 00000000000..675e615aecc
--- /dev/null
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/replicaset_expansion.go
@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ "fmt"
+
+ apps "k8s.io/api/apps/v1"
+ "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+)
+
+// ReplicaSetListerExpansion allows custom methods to be added to
+// ReplicaSetLister.
+type ReplicaSetListerExpansion interface {
+ GetPodReplicaSets(pod *v1.Pod) ([]*apps.ReplicaSet, error)
+}
+
+// ReplicaSetNamespaceListerExpansion allows custom methods to be added to
+// ReplicaSetNamespaceLister.
+type ReplicaSetNamespaceListerExpansion interface{}
+
+// GetPodReplicaSets returns a list of ReplicaSets that potentially match a pod.
+// Only the one specified in the Pod's ControllerRef will actually manage it.
+// Returns an error only if no matching ReplicaSets are found.
+func (s *replicaSetLister) GetPodReplicaSets(pod *v1.Pod) ([]*apps.ReplicaSet, error) {
+ if len(pod.Labels) == 0 {
+ return nil, fmt.Errorf("no ReplicaSets found for pod %v because it has no labels", pod.Name)
+ }
+
+ list, err := s.ReplicaSets(pod.Namespace).List(labels.Everything())
+ if err != nil {
+ return nil, err
+ }
+
+ var rss []*apps.ReplicaSet
+ for _, rs := range list {
+ if rs.Namespace != pod.Namespace {
+ continue
+ }
+ selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+ if err != nil {
+ return nil, fmt.Errorf("invalid selector: %v", err)
+ }
+
+ // If a ReplicaSet with a nil or empty selector creeps in, it should match nothing, not everything.
+ if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
+ continue
+ }
+ rss = append(rss, rs)
+ }
+
+ if len(rss) == 0 {
+ return nil, fmt.Errorf("could not find ReplicaSet for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
+ }
+
+ return rss, nil
+}
diff --git a/staging/src/k8s.io/client-go/listers/apps/v1/statefulset_expansion.go b/staging/src/k8s.io/client-go/listers/apps/v1/statefulset_expansion.go
new file mode 100644
index 00000000000..b4912976b69
--- /dev/null
+++ b/staging/src/k8s.io/client-go/listers/apps/v1/statefulset_expansion.go
@@ -0,0 +1,77 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ "fmt"
+
+ apps "k8s.io/api/apps/v1"
+ "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+)
+
+// StatefulSetListerExpansion allows custom methods to be added to
+// StatefulSetLister.
+type StatefulSetListerExpansion interface {
+ GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error)
+}
+
+// StatefulSetNamespaceListerExpansion allows custom methods to be added to
+// StatefulSetNamespaceLister.
+type StatefulSetNamespaceListerExpansion interface{}
+
+// GetPodStatefulSets returns a list of StatefulSets that potentially match a pod.
+// Only the one specified in the Pod's ControllerRef will actually manage it.
+// Returns an error only if no matching StatefulSets are found.
+func (s *statefulSetLister) GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error) {
+ var selector labels.Selector
+ var ps *apps.StatefulSet
+
+ if len(pod.Labels) == 0 {
+ return nil, fmt.Errorf("no StatefulSets found for pod %v because it has no labels", pod.Name)
+ }
+
+ list, err := s.StatefulSets(pod.Namespace).List(labels.Everything())
+ if err != nil {
+ return nil, err
+ }
+
+ var psList []*apps.StatefulSet
+ for i := range list {
+ ps = list[i]
+ if ps.Namespace != pod.Namespace {
+ continue
+ }
+ selector, err = metav1.LabelSelectorAsSelector(ps.Spec.Selector)
+ if err != nil {
+ return nil, fmt.Errorf("invalid selector: %v", err)
+ }
+
+ // If a StatefulSet with a nil or empty selector creeps in, it should match nothing, not everything.
+ if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
+ continue
+ }
+ psList = append(psList, ps)
+ }
+
+ if len(psList) == 0 {
+ return nil, fmt.Errorf("could not find StatefulSet for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
+ }
+
+ return psList, nil
+}
diff --git a/test/e2e/scalability/density.go b/test/e2e/scalability/density.go
index f4b9c0b8825..5586fadcae9 100644
--- a/test/e2e/scalability/density.go
+++ b/test/e2e/scalability/density.go
@@ -200,7 +200,7 @@ func density30AddonResourceVerifier(numNodes int) map[string]framework.ResourceC
MemoryConstraint: 100 * (1024 * 1024),
}
constraints["l7-lb-controller"] = framework.ResourceConstraint{
- CPUConstraint: 0.2 + 0.0001*float64(numNodes),
+ CPUConstraint: 0.2 + 0.00015*float64(numNodes),
MemoryConstraint: (75 + uint64(math.Ceil(0.8*float64(numNodes)))) * (1024 * 1024),
}
constraints["influxdb"] = framework.ResourceConstraint{