github/kubernetes
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into upgrade_aliases_branch

Browse Source
This commit is contained in:
Jing Ai
2017-11-23 17:41:49 -08:00
parent a538c1e82e 362e6293ce
commit b40793e7dd
98 changed files with 3319 additions and 590 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
api/openapi-spec/swagger.json generated
View File

@@ -79217,6 +79217,18 @@
}
}
},
"io.k8s.api.extensions.v1beta1.AllowedFlexVolume": {
"description": "AllowedFlexVolume represents a single Flexvolume that is allowed to be used.",
"required": [
"driver"
],
"properties": {
"driver": {
"description": "Driver is the name of the Flexvolume driver.",
"type": "string"
}
}
},
"io.k8s.api.extensions.v1beta1.AllowedHostPath": {
"description": "defines the host volume conditions that will be enabled by a policy for pods to use. It requires the path prefix to be defined.",
"properties": {
@@ -80171,6 +80183,13 @@
"type": "string"
}
},
"allowedFlexVolumes": {
"description": "AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the \"Volumes\" field.",
"type": "array",
"items": {
"$ref": "#/definitions/io.k8s.api.extensions.v1beta1.AllowedFlexVolume"
}
},
"allowedHostPaths": {
"description": "is a white list of allowed host paths. Empty indicates that all host paths may be used.",
"type": "array",

20
api/swagger-spec/extensions_v1beta1.json generated
View File

@@ -10314,6 +10314,13 @@
"$ref": "v1beta1.AllowedHostPath"
},
"description": "is a white list of allowed host paths. Empty indicates that all host paths may be used."
},
"allowedFlexVolumes": {
"type": "array",
"items": {
"$ref": "v1beta1.AllowedFlexVolume"
},
"description": "AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the \"Volumes\" field."
}
}
},
@@ -10442,6 +10449,19 @@
}
}
},
"v1beta1.AllowedFlexVolume": {
"id": "v1beta1.AllowedFlexVolume",
"description": "AllowedFlexVolume represents a single Flexvolume that is allowed to be used.",
"required": [
"driver"
],
"properties": {
"driver": {
"type": "string",
"description": "Driver is the name of the Flexvolume driver."
}
}
},
"v1beta1.ReplicaSetList": {
"id": "v1beta1.ReplicaSetList",
"description": "ReplicaSetList is a collection of ReplicaSets.",

8
build/debian-iptables/Dockerfile
View File

@@ -19,8 +19,8 @@ FROM BASEIMAGE
CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
RUN clean-install \
iptables \
ebtables \
conntrack \
module-init-tools \
ipset
ebtables \
ipset \
iptables \
kmod

27
build/lib/release.sh
View File

@@ -98,18 +98,23 @@ function kube::release::package_tarballs() {
# Package the source code we built, for compliance/licensing/audit/yadda.
function kube::release::package_src_tarball() {
local -r src_tarball="${RELEASE_TARS}/kubernetes-src.tar.gz"
kube::log::status "Building tarball: src"
local source_files=(
$(cd "${KUBE_ROOT}" && find . -mindepth 1 -maxdepth 1 \
-not \( \
\( -path ./_\* -o \
-path ./.git\* -o \
-path ./.config\* -o \
-path ./.gsutil\* \
\) -prune \
\))
)
"${TAR}" czf "${RELEASE_TARS}/kubernetes-src.tar.gz" -C "${KUBE_ROOT}" "${source_files[@]}"
if [[ "${KUBE_GIT_TREE_STATE-}" == "clean" ]]; then
git archive -o "${src_tarball}" HEAD
else
local source_files=(
$(cd "${KUBE_ROOT}" && find . -mindepth 1 -maxdepth 1 \
-not \( \
\( -path ./_\* -o \
-path ./.git\* -o \
-path ./.config\* -o \
-path ./.gsutil\* \
\) -prune \
\))
)
"${TAR}" czf "${src_tarball}" -C "${KUBE_ROOT}" "${source_files[@]}"
fi
}
# Package up all of the cross compiled clients. Over time this should grow into

6
cluster/addons/metadata-agent/OWNERS Normal file
View File

@@ -0,0 +1,6 @@
approvers:
- kawych
- piosz
reviewers:
- kawych
- piosz

4
cluster/addons/metadata-agent/README.md Normal file
View File

@@ -0,0 +1,4 @@
# Kubernetes Metadata Agent
Metadata Agent is a source of metadata required by logging and monitoring agents
running on a cluster.

38
cluster/addons/metadata-agent/stackdriver/metadata-agent.yaml Normal file
View File

@@ -0,0 +1,38 @@
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
labels:
app: stackdriver-agents
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
name: stackdriver-agents
namespace: kube-system
spec:
selector:
matchLabels:
app: stackdriver-agents
template:
metadata:
labels:
app: stackdriver-agents
spec:
containers:
- image: us.gcr.io/container-monitoring-storage/stackdriver-metadata-agent:{{ metadata_agent_version }}
imagePullPolicy: IfNotPresent
name: metadata-agent
ports:
- containerPort: 8000
hostPort: 8000
protocol: TCP
resources:
requests:
cpu: {{ metadata_agent_cpu_request }}
memory: {{ metadata_agent_memory_request }}
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
terminationGracePeriodSeconds: 30
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate

2
cluster/centos/config-default.sh
View File

@@ -120,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"}
# Admission Controllers to invoke prior to persisting objects in cluster
# If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,ResourceQuota"}
export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,PVCProtection,ResourceQuota"}
# Extra options to set on the Docker command line.
# This is useful for setting --insecure-registry for local registries.

2
cluster/common.sh
View File

@@ -610,6 +610,8 @@ KUBERNETES_MASTER_NAME: $(yaml-quote ${KUBERNETES_MASTER_NAME})
ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})
ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})
ENABLE_METRICS_SERVER: $(yaml-quote ${ENABLE_METRICS_SERVER:-false})
ENABLE_METADATA_AGENT: $(yaml-quote ${ENABLE_METADATA_AGENT:-none})
METADATA_AGENT_VERSION: $(yaml-quote ${METADATA_AGENT_VERSION:-})
DOCKER_REGISTRY_MIRROR_URL: $(yaml-quote ${DOCKER_REGISTRY_MIRROR_URL:-})
ENABLE_L7_LOADBALANCING: $(yaml-quote ${ENABLE_L7_LOADBALANCING:-none})
ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})

12
cluster/gce/config-default.sh
View File

@@ -150,6 +150,16 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
# TODO(piosz) remove this option once Metrics Server became a stable thing.
ENABLE_METRICS_SERVER="${KUBE_ENABLE_METRICS_SERVER:-true}"
# Optional: Metadata agent to setup as part of the cluster bring up:
# none - No metadata agent
# stackdriver - Stackdriver metadata agent
# Metadata agent is a daemon set that provides metadata of kubernetes objects
# running on the same node for exporting metrics and logs.
ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}"
# Version tag of metadata agent
METADATA_AGENT_VERSION="${KUBE_METADATA_AGENT_VERSION:-0.2-0.0.13-5}"
# One special node out of NUM_NODES would be created of this type if specified.
# Useful for scheduling heapster in large clusters with nodes of small size.
HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
@@ -289,7 +299,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
fi
# Admission Controllers to invoke prior to persisting objects in cluster
ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority
ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,PVCProtection
if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy"

14
cluster/gce/container-linux/configure-helper.sh
View File

@@ -1297,6 +1297,20 @@ EOF
sed -i -e "s@{{ use_new_resources }}@${use_new_resources}@g" "${controller_yaml}"
fi
fi
if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] ||
([[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]] &&
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]); then
if [[ "${ENABLE_METADATA_AGENT:-}" == "stackdriver" ]] &&
[[ "${METADATA_AGENT_VERSION:-}" != "" ]]; then
metadata_agent_cpu_request="${METADATA_AGENT_CPU_REQUEST:-40m}"
metadata_agent_memory_request="${METADATA_AGENT_MEMORY_REQUEST:-50Mi}"
setup-addon-manifests "addons" "metadata-agent/stackdriver"
deployment_yaml="${dst_dir}/metadata-agent/stackdriver/metadata-agent.yaml"
sed -i -e "s@{{ metadata_agent_version }}@${METADATA_AGENT_VERSION}@g" "${deployment_yaml}"
sed -i -e "s@{{ metadata_agent_cpu_request }}@${metadata_agent_cpu_request}@g" "${deployment_yaml}"
sed -i -e "s@{{ metadata_agent_memory_request }}@${metadata_agent_memory_request}@g" "${deployment_yaml}"
fi
fi
if [[ "${ENABLE_METRICS_SERVER:-}" == "true" ]]; then
setup-addon-manifests "addons" "metrics-server"
fi

14
cluster/gce/gci/configure-helper.sh
View File

@@ -2076,6 +2076,20 @@ EOF
sed -i -e "s@{{ use_new_resources }}@${use_new_resources}@g" "${controller_yaml}"
fi
fi
if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] ||
([[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]] &&
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]); then
if [[ "${ENABLE_METADATA_AGENT:-}" == "stackdriver" ]] &&
[[ "${METADATA_AGENT_VERSION:-}" != "" ]]; then
metadata_agent_cpu_request="${METADATA_AGENT_CPU_REQUEST:-40m}"
metadata_agent_memory_request="${METADATA_AGENT_MEMORY_REQUEST:-50Mi}"
setup-addon-manifests "addons" "metadata-agent/stackdriver"
deployment_yaml="${dst_dir}/metadata-agent/stackdriver/metadata-agent.yaml"
sed -i -e "s@{{ metadata_agent_version }}@${METADATA_AGENT_VERSION}@g" "${deployment_yaml}"
sed -i -e "s@{{ metadata_agent_cpu_request }}@${metadata_agent_cpu_request}@g" "${deployment_yaml}"
sed -i -e "s@{{ metadata_agent_memory_request }}@${metadata_agent_memory_request}@g" "${deployment_yaml}"
fi
fi
if [[ "${ENABLE_METRICS_SERVER:-}" == "true" ]]; then
setup-addon-manifests "addons" "metrics-server"
fi

10
cluster/gce/util.sh
View File

@@ -1526,14 +1526,20 @@ function check-cluster() {
fi
local start_time=$(date +%s)
local curl_out=$(mktemp)
kube::util::trap_add "rm -f ${curl_out}" EXIT
until curl --cacert "${CERT_DIR}/pki/ca.crt" \
-H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \
${secure} \
--max-time 5 --fail --output /dev/null --silent \
"https://${KUBE_MASTER_IP}/api/v1/pods"; do
--max-time 5 --fail \
"https://${KUBE_MASTER_IP}/api/v1/pods" > "${curl_out}" 2>&1; do
local elapsed=$(($(date +%s) - ${start_time}))
if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then
echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2
echo "Last output from querying API server follows:" >&2
echo "-----------------------------------------------------" >&2
cat "${curl_out}" >&2
echo "-----------------------------------------------------" >&2
exit 2
fi
printf "."

2
cluster/libvirt-coreos/util.sh
View File

@@ -27,7 +27,7 @@ source "$KUBE_ROOT/cluster/common.sh"
export LIBVIRT_DEFAULT_URI=qemu:///system
export SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true}
export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,ResourceQuota}
export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,PVCProtection,ResourceQuota}
readonly POOL=kubernetes
readonly POOL_PATH=/var/lib/libvirt/images/kubernetes

3
cluster/vagrant/config-default.sh
View File

@@ -56,7 +56,7 @@ MASTER_PASSWD="${MASTER_PASSWD:-vagrant}"
# Admission Controllers to invoke prior to persisting objects in cluster
# If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota
ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,PVCProtection,ResourceQuota
# Optional: Enable node logging.
ENABLE_NODE_LOGGING=false
@@ -120,4 +120,3 @@ E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
# Default fallback NETWORK_IF_NAME, will be used in case when no 'VAGRANT-BEGIN' comments were defined in network-script
export DEFAULT_NETWORK_IF_NAME="eth0"

1
cmd/kube-apiserver/app/options/BUILD
View File

@@ -40,6 +40,7 @@ go_library(
"//plugin/pkg/admission/noderestriction:go_default_library",
"//plugin/pkg/admission/persistentvolume/label:go_default_library",
"//plugin/pkg/admission/persistentvolume/resize:go_default_library",
"//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library",
"//plugin/pkg/admission/podnodeselector:go_default_library",
"//plugin/pkg/admission/podpreset:go_default_library",
"//plugin/pkg/admission/podtolerationrestriction:go_default_library",

2
cmd/kube-apiserver/app/options/plugins.go
View File

@@ -42,6 +42,7 @@ import (
"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize"
"k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection"
"k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
"k8s.io/kubernetes/plugin/pkg/admission/podpreset"
"k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction"
@@ -81,4 +82,5 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
serviceaccount.Register(plugins)
setdefault.Register(plugins)
resize.Register(plugins)
pvcprotection.Register(plugins)
}

7
cmd/kube-controller-manager/app/BUILD
View File

@@ -43,12 +43,6 @@ go_library(
"//pkg/apis/storage/install:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers:go_default_library",
"//pkg/cloudprovider/providers/aws:go_default_library",
"//pkg/cloudprovider/providers/azure:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/cloudprovider/providers/openstack:go_default_library",
"//pkg/cloudprovider/providers/photon:go_default_library",
"//pkg/cloudprovider/providers/vsphere:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/controller/bootstrap:go_default_library",
"//pkg/controller/certificates/approver:go_default_library",
@@ -79,6 +73,7 @@ go_library(
"//pkg/controller/volume/attachdetach:go_default_library",
"//pkg/controller/volume/expand:go_default_library",
"//pkg/controller/volume/persistentvolume:go_default_library",
"//pkg/controller/volume/pvcprotection:go_default_library",
"//pkg/features:go_default_library",
"//pkg/quota/generic:go_default_library",
"//pkg/quota/install:go_default_library",

1
cmd/kube-controller-manager/app/controllermanager.go
View File

@@ -359,6 +359,7 @@ func NewControllerInitializers() map[string]InitFunc {
controllers["attachdetach"] = startAttachDetachController
controllers["persistentvolume-expander"] = startVolumeExpandController
controllers["clusterrole-aggregation"] = startClusterRoleAggregrationController
controllers["pvc-protection"] = startPVCProtectionController
return controllers
}

13
cmd/kube-controller-manager/app/core.go
View File

@@ -53,6 +53,7 @@ import (
"k8s.io/kubernetes/pkg/controller/volume/attachdetach"
"k8s.io/kubernetes/pkg/controller/volume/expand"
persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
"k8s.io/kubernetes/pkg/controller/volume/pvcprotection"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/quota/generic"
quotainstall "k8s.io/kubernetes/pkg/quota/install"
@@ -376,3 +377,15 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
return true, nil
}
func startPVCProtectionController(ctx ControllerContext) (bool, error) {
if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
go pvcprotection.NewPVCProtectionController(
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ctx.InformerFactory.Core().V1().Pods(),
ctx.ClientBuilder.ClientOrDie("pvc-protection-controller"),
).Run(1, ctx.Stop)
return true, nil
}
return false, nil
}

28
cmd/kube-controller-manager/app/plugins.go
View File

@@ -30,12 +30,6 @@ import (
// Volume plugins
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
"k8s.io/kubernetes/pkg/cloudprovider/providers/photon"
"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/aws_ebs"
"k8s.io/kubernetes/pkg/volume/azure_dd"
@@ -165,22 +159,12 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
if cloud != nil {
switch {
case aws.ProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...)
case gce.ProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...)
case openstack.ProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
case vsphere.ProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
case azure.CloudProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
case photon.ProviderName == cloud.ProviderName():
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
}
}
allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
return allPlugins
}

2
cmd/kube-proxy/app/server.go
View File

@@ -218,7 +218,7 @@ func (o *Options) Run() error {
return o.writeConfigFile()
}
proxyServer, err := NewProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)
proxyServer, err := NewProxyServer(o)
if err != nil {
return err
}

12
cmd/kube-proxy/app/server_others.go
View File

@@ -54,7 +54,17 @@ import (
)
// NewProxyServer returns a new ProxyServer.
func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, cleanupIPVS bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) {
func NewProxyServer(o *Options) (*ProxyServer, error) {
return newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)
}
func newProxyServer(
config *proxyconfigapi.KubeProxyConfiguration,
cleanupAndExit bool,
cleanupIPVS bool,
scheme *runtime.Scheme,
master string) (*ProxyServer, error) {
if config == nil {
return nil, errors.New("config is required")
}

2
cmd/kube-proxy/app/server_test.go
View File

@@ -162,7 +162,7 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) {
}
options.CleanupAndExit = true
proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.CleanupIPVS, options.scheme, options.master)
proxyserver, err := NewProxyServer(options)
assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)

6
cmd/kube-proxy/app/server_windows.go
View File

@@ -46,7 +46,11 @@ import (
)
// NewProxyServer returns a new ProxyServer.
func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) {
func NewProxyServer(o *Options) (*ProxyServer, error) {
return newProxyServer(o.config, o.CleanupAndExit, o.scheme, o.master)
}
func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) {
if config == nil {
return nil, errors.New("config is required")
}

11
cmd/kubeadm/app/apis/kubeadm/validation/validation.go
View File

@@ -331,25 +331,22 @@ func ValidateAPIEndpoint(c *kubeadm.MasterConfiguration, fldPath *field.Path) fi
return allErrs
}
// ValidateIgnoreChecksErrors validates duplicates in ignore-checks-errors flag.
func ValidateIgnoreChecksErrors(ignoreChecksErrors []string, skipPreflightChecks bool) (sets.String, error) {
// ValidateIgnorePreflightErrors validates duplicates in ignore-preflight-errors flag.
func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflightChecks bool) (sets.String, error) {
ignoreErrors := sets.NewString()
allErrs := field.ErrorList{}
for _, item := range ignoreChecksErrors {
for _, item := range ignorePreflightErrors {
ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive
}
// TODO: remove once deprecated flag --skip-preflight-checks is removed.
if skipPreflightChecks {
if ignoreErrors.Has("all") {
allErrs = append(allErrs, field.Invalid(field.NewPath("ignore-checks-errors"), strings.Join(ignoreErrors.List(), ","), "'all' is used together with deprecated flag --skip-preflight-checks. Remove deprecated flag"))
}
ignoreErrors.Insert("all")
}
if ignoreErrors.Has("all") && ignoreErrors.Len() > 1 {
allErrs = append(allErrs, field.Invalid(field.NewPath("ignore-checks-errors"), strings.Join(ignoreErrors.List(), ","), "don't specify individual checks if 'all' is used"))
allErrs = append(allErrs, field.Invalid(field.NewPath("ignore-preflight-errors"), strings.Join(ignoreErrors.List(), ","), "don't specify individual checks if 'all' is used"))
}
return ignoreErrors, allErrs.ToAggregate()

20
cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go
View File

@@ -459,12 +459,12 @@ func TestValidateFeatureGates(t *testing.T) {
}
}
func TestValidateIgnoreChecksErrors(t *testing.T) {
func TestValidateIgnorePreflightErrors(t *testing.T) {
var tests = []struct {
ignoreChecksErrors []string
skipPreflightChecks bool
expectedLen int
expectedError bool
ignorePreflightErrors []string
skipPreflightChecks bool
expectedLen int
expectedError bool
}{
{[]string{}, false, 0, false}, // empty list, no old skip-preflight-checks
{[]string{}, true, 1, false}, // empty list, old skip-preflight-checks
@@ -473,17 +473,17 @@ func TestValidateIgnoreChecksErrors(t *testing.T) {
{[]string{"check1", "check2", "check1"}, false, 2, false}, // duplicates
{[]string{"check1", "check2", "all"}, false, 3, true}, // non-duplicate, but 'all' present together wth individual checks
{[]string{"all"}, false, 1, false}, // skip all checks by using new flag
{[]string{"all"}, true, 1, true}, // skip all checks by using both old and new flags at the same time
{[]string{"all"}, true, 1, false}, // skip all checks by using both old and new flags at the same time
}
for _, rt := range tests {
result, err := ValidateIgnoreChecksErrors(rt.ignoreChecksErrors, rt.skipPreflightChecks)
result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrors, rt.skipPreflightChecks)
switch {
case err != nil && !rt.expectedError:
t.Errorf("ValidateIgnoreChecksErrors: unexpected error for input (%s, %v), error: %v", rt.ignoreChecksErrors, rt.skipPreflightChecks, err)
t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s, %v), error: %v", rt.ignorePreflightErrors, rt.skipPreflightChecks, err)
case err == nil && rt.expectedError:
t.Errorf("ValidateIgnoreChecksErrors: expected error for input (%s, %v) but got: %v", rt.ignoreChecksErrors, rt.skipPreflightChecks, result)
t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s, %v) but got: %v", rt.ignorePreflightErrors, rt.skipPreflightChecks, result)
case result.Len() != rt.expectedLen:
t.Errorf("ValidateIgnoreChecksErrors: expected Len = %d for input (%s, %v) but got: %v, %v", rt.expectedLen, rt.ignoreChecksErrors, rt.skipPreflightChecks, result.Len(), result)
t.Errorf("ValidateIgnorePreflightErrors: expected Len = %d for input (%s, %v) but got: %v, %v", rt.expectedLen, rt.ignorePreflightErrors, rt.skipPreflightChecks, result.Len(), result)
}
}
}

20
cmd/kubeadm/app/cmd/init.go
View File

@@ -113,7 +113,7 @@ func NewCmdInit(out io.Writer) *cobra.Command {
var dryRun bool
var featureGatesString string
var criSocket string
var ignoreChecksErrors []string
var ignorePreflightErrors []string
cmd := &cobra.Command{
Use: "init",
@@ -128,10 +128,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
internalcfg := &kubeadmapi.MasterConfiguration{}
legacyscheme.Scheme.Convert(cfg, internalcfg, nil)
ignoreChecksErrorsSet, err := validation.ValidateIgnoreChecksErrors(ignoreChecksErrors, skipPreFlight)
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors, skipPreFlight)
kubeadmutil.CheckErr(err)
i, err := NewInit(cfgPath, internalcfg, ignoreChecksErrorsSet, skipTokenPrint, dryRun, criSocket)
i, err := NewInit(cfgPath, internalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun, criSocket)
kubeadmutil.CheckErr(err)
kubeadmutil.CheckErr(i.Validate(cmd))
kubeadmutil.CheckErr(i.Run(out))
@@ -139,7 +139,7 @@ func NewCmdInit(out io.Writer) *cobra.Command {
}
AddInitConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString)
AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun, &criSocket, &ignoreChecksErrors)
AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun, &criSocket, &ignorePreflightErrors)
return cmd
}
@@ -195,13 +195,13 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.MasterConfigur
}
// AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset
func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, skipTokenPrint, dryRun *bool, criSocket *string, ignoreChecksErrors *[]string) {
func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, skipTokenPrint, dryRun *bool, criSocket *string, ignorePreflightErrors *[]string) {
flagSet.StringVar(
cfgPath, "config", *cfgPath,
"Path to kubeadm config file. WARNING: Usage of a configuration file is experimental.",
)
flagSet.StringSliceVar(
ignoreChecksErrors, "ignore-checks-errors", *ignoreChecksErrors,
ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors,
"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
)
// Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
@@ -209,7 +209,7 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk
skipPreFlight, "skip-preflight-checks", *skipPreFlight,
"Skip preflight checks which normally run before modifying the system.",
)
flagSet.MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-checks-errors=all")
flagSet.MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-preflight-errors=all")
// Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
flagSet.BoolVar(
skipTokenPrint, "skip-token-print", *skipTokenPrint,
@@ -227,7 +227,7 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk
}
// NewInit validates given arguments and instantiates Init struct with provided information.
func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, ignoreChecksErrors sets.String, skipTokenPrint, dryRun bool, criSocket string) (*Init, error) {
func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, ignorePreflightErrors sets.String, skipTokenPrint, dryRun bool, criSocket string) (*Init, error) {
fmt.Println("[kubeadm] WARNING: kubeadm is currently in beta")
if cfgPath != "" {
@@ -261,12 +261,12 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, ignoreChecksEr
fmt.Println("[preflight] Running pre-flight checks.")
if err := preflight.RunInitMasterChecks(utilsexec.New(), cfg, criSocket, ignoreChecksErrors); err != nil {
if err := preflight.RunInitMasterChecks(utilsexec.New(), cfg, criSocket, ignorePreflightErrors); err != nil {
return nil, err
}
// Try to start the kubelet service in case it's inactive
preflight.TryStartKubelet(ignoreChecksErrors)
preflight.TryStartKubelet(ignorePreflightErrors)
return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun}, nil
}

20
cmd/kubeadm/app/cmd/join.go
View File

@@ -108,7 +108,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
var cfgPath string
var criSocket string
var featureGatesString string
var ignoreChecksErrors []string
var ignorePreflightErrors []string
cmd := &cobra.Command{
Use: "join [flags]",
@@ -126,10 +126,10 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
internalcfg := &kubeadmapi.NodeConfiguration{}
legacyscheme.Scheme.Convert(cfg, internalcfg, nil)
ignoreChecksErrorsSet, err := validation.ValidateIgnoreChecksErrors(ignoreChecksErrors, skipPreFlight)
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors, skipPreFlight)
kubeadmutil.CheckErr(err)
j, err := NewJoin(cfgPath, args, internalcfg, ignoreChecksErrorsSet, criSocket)
j, err := NewJoin(cfgPath, args, internalcfg, ignorePreflightErrorsSet, criSocket)
kubeadmutil.CheckErr(err)
kubeadmutil.CheckErr(j.Validate(cmd))
kubeadmutil.CheckErr(j.Run(out))
@@ -137,7 +137,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
}
AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString)
AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &criSocket, &ignoreChecksErrors)
AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &criSocket, &ignorePreflightErrors)
return cmd
}
@@ -172,20 +172,20 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfigurat
}
// AddJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset
func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight *bool, criSocket *string, ignoreChecksErrors *[]string) {
func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight *bool, criSocket *string, ignorePreflightErrors *[]string) {
flagSet.StringVar(
cfgPath, "config", *cfgPath,
"Path to kubeadm config file.")
flagSet.StringSliceVar(
ignoreChecksErrors, "ignore-checks-errors", *ignoreChecksErrors,
ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors,
"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
)
flagSet.BoolVar(
skipPreFlight, "skip-preflight-checks", false,
"Skip preflight checks which normally run before modifying the system.",
)
flagSet.MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-checks-errors=all")
flagSet.MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-preflight-errors=all")
flagSet.StringVar(
criSocket, "cri-socket", "/var/run/dockershim.sock",
`Specify the CRI socket to connect to.`,
@@ -198,7 +198,7 @@ type Join struct {
}
// NewJoin instantiates Join struct with given arguments
func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, ignoreChecksErrors sets.String, criSocket string) (*Join, error) {
func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, ignorePreflightErrors sets.String, criSocket string) (*Join, error) {
fmt.Println("[kubeadm] WARNING: kubeadm is currently in beta")
if cfg.NodeName == "" {
@@ -218,12 +218,12 @@ func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, i
fmt.Println("[preflight] Running pre-flight checks.")
// Then continue with the others...
if err := preflight.RunJoinNodeChecks(utilsexec.New(), cfg, criSocket, ignoreChecksErrors); err != nil {
if err := preflight.RunJoinNodeChecks(utilsexec.New(), cfg, criSocket, ignorePreflightErrors); err != nil {
return nil, err
}
// Try to start the kubelet service in case it's inactive
preflight.TryStartKubelet(ignoreChecksErrors)
preflight.TryStartKubelet(ignorePreflightErrors)
return &Join{cfg: cfg}, nil
}

14
cmd/kubeadm/app/cmd/reset.go
View File

@@ -47,30 +47,30 @@ func NewCmdReset(out io.Writer) *cobra.Command {
var skipPreFlight bool
var certsDir string
var criSocketPath string
var ignoreChecksErrors []string
var ignorePreflightErrors []string
cmd := &cobra.Command{
Use: "reset",
Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.",
Run: func(cmd *cobra.Command, args []string) {
ignoreChecksErrorsSet, err := validation.ValidateIgnoreChecksErrors(ignoreChecksErrors, skipPreFlight)
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors, skipPreFlight)
kubeadmutil.CheckErr(err)
r, err := NewReset(ignoreChecksErrorsSet, certsDir, criSocketPath)
r, err := NewReset(ignorePreflightErrorsSet, certsDir, criSocketPath)
kubeadmutil.CheckErr(err)
kubeadmutil.CheckErr(r.Run(out))
},
}
cmd.PersistentFlags().StringSliceVar(
&ignoreChecksErrors, "ignore-checks-errors", ignoreChecksErrors,
&ignorePreflightErrors, "ignore-preflight-errors", ignorePreflightErrors,
"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
)
cmd.PersistentFlags().BoolVar(
&skipPreFlight, "skip-preflight-checks", false,
"Skip preflight checks which normally run before modifying the system.",
)
cmd.PersistentFlags().MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-checks-errors=all")
cmd.PersistentFlags().MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-preflight-errors=all")
cmd.PersistentFlags().StringVar(
&certsDir, "cert-dir", kubeadmapiext.DefaultCertificatesDir,
@@ -92,10 +92,10 @@ type Reset struct {
}
// NewReset instantiate Reset struct
func NewReset(ignoreChecksErrors sets.String, certsDir, criSocketPath string) (*Reset, error) {
func NewReset(ignorePreflightErrors sets.String, certsDir, criSocketPath string) (*Reset, error) {
fmt.Println("[preflight] Running pre-flight checks.")
if err := preflight.RunRootCheckOnly(ignoreChecksErrors); err != nil {
if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil {
return nil, err
}

4
cmd/kubeadm/app/cmd/upgrade/apply.go
View File

@@ -72,11 +72,11 @@ func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command {
Short: "Upgrade your Kubernetes cluster to the specified version.",
Run: func(cmd *cobra.Command, args []string) {
var err error
flags.parent.ignoreChecksErrorsSet, err = validation.ValidateIgnoreChecksErrors(flags.parent.ignoreChecksErrors, flags.parent.skipPreFlight)
flags.parent.ignorePreflightErrorsSet, err = validation.ValidateIgnorePreflightErrors(flags.parent.ignorePreflightErrors, flags.parent.skipPreFlight)
kubeadmutil.CheckErr(err)
// Ensure the user is root
err = runPreflightChecks(flags.parent.ignoreChecksErrorsSet)
err = runPreflightChecks(flags.parent.ignorePreflightErrorsSet)
kubeadmutil.CheckErr(err)
err = cmdutil.ValidateExactArgNumber(args, []string{"version"})

4
cmd/kubeadm/app/cmd/upgrade/common.go
View File

@@ -107,9 +107,9 @@ func printConfiguration(cfg *kubeadmapiext.MasterConfiguration, w io.Writer) {
}
// runPreflightChecks runs the root preflight check
func runPreflightChecks(ignoreChecksErrors sets.String) error {
func runPreflightChecks(ignorePreflightErrors sets.String) error {
fmt.Println("[preflight] Running pre-flight checks.")
return preflight.RunRootCheckOnly(ignoreChecksErrors)
return preflight.RunRootCheckOnly(ignorePreflightErrors)
}
// getClient gets a real or fake client depending on whether the user is dry-running or not

4
cmd/kubeadm/app/cmd/upgrade/plan.go
View File

@@ -38,10 +38,10 @@ func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command {
Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable.",
Run: func(_ *cobra.Command, _ []string) {
var err error
parentFlags.ignoreChecksErrorsSet, err = validation.ValidateIgnoreChecksErrors(parentFlags.ignoreChecksErrors, parentFlags.skipPreFlight)
parentFlags.ignorePreflightErrorsSet, err = validation.ValidateIgnorePreflightErrors(parentFlags.ignorePreflightErrors, parentFlags.skipPreFlight)
kubeadmutil.CheckErr(err)
// Ensure the user is root
err = runPreflightChecks(parentFlags.ignoreChecksErrorsSet)
err = runPreflightChecks(parentFlags.ignorePreflightErrorsSet)
kubeadmutil.CheckErr(err)
err = RunPlan(parentFlags)

10
cmd/kubeadm/app/cmd/upgrade/upgrade.go
View File

@@ -35,8 +35,8 @@ type cmdUpgradeFlags struct {
allowRCUpgrades bool
printConfig bool
skipPreFlight bool
ignoreChecksErrors []string
ignoreChecksErrorsSet sets.String
ignorePreflightErrors []string
ignorePreflightErrorsSet sets.String
}
// NewCmdUpgrade returns the cobra command for `kubeadm upgrade`
@@ -49,7 +49,7 @@ func NewCmdUpgrade(out io.Writer) *cobra.Command {
allowRCUpgrades: false,
printConfig: false,
skipPreFlight: false,
ignoreChecksErrorsSet: sets.NewString(),
ignorePreflightErrorsSet: sets.NewString(),
}
cmd := &cobra.Command{
@@ -63,9 +63,9 @@ func NewCmdUpgrade(out io.Writer) *cobra.Command {
cmd.PersistentFlags().BoolVar(&flags.allowExperimentalUpgrades, "allow-experimental-upgrades", flags.allowExperimentalUpgrades, "Show unstable versions of Kubernetes as an upgrade alternative and allow upgrading to an alpha/beta/release candidate versions of Kubernetes.")
cmd.PersistentFlags().BoolVar(&flags.allowRCUpgrades, "allow-release-candidate-upgrades", flags.allowRCUpgrades, "Show release candidate versions of Kubernetes as an upgrade alternative and allow upgrading to a release candidate versions of Kubernetes.")
cmd.PersistentFlags().BoolVar(&flags.printConfig, "print-config", flags.printConfig, "Specifies whether the configuration file that will be used in the upgrade should be printed or not.")
cmd.PersistentFlags().StringSliceVar(&flags.ignoreChecksErrors, "ignore-checks-errors", flags.ignoreChecksErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
cmd.PersistentFlags().StringSliceVar(&flags.ignorePreflightErrors, "ignore-preflight-errors", flags.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
cmd.PersistentFlags().BoolVar(&flags.skipPreFlight, "skip-preflight-checks", flags.skipPreFlight, "Skip preflight checks that normally run before modifying the system.")
cmd.PersistentFlags().MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-checks-errors=all")
cmd.PersistentFlags().MarkDeprecated("skip-preflight-checks", "it is now equivalent to --ignore-preflight-errors=all")
cmd.PersistentFlags().StringVar(&flags.featureGatesString, "feature-gates", flags.featureGatesString, "A set of key=value pairs that describe feature gates for various features."+
"Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n"))

2
cmd/kubeadm/app/constants/constants.go
View File

@@ -236,7 +236,7 @@ var (
SupportedEtcdVersion = map[uint8]string{
8: "3.0.17",
9: "3.1.10",
10: "3.1.11",
10: "3.1.10",
}
)

2
cmd/kubeadm/app/constants/constants_test.go
View File

@@ -137,7 +137,7 @@ func TestEtcdSupportedVersion(t *testing.T) {
},
{
kubernetesVersion: "1.10.0",
expectedVersion: version.MustParseSemantic("3.1.11"),
expectedVersion: version.MustParseSemantic("3.1.10"),
expectedError: nil,
},
{

9
cmd/kubeadm/app/phases/controlplane/volumes_test.go
View File

@@ -513,6 +513,15 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
for _, rt := range tests {
mounts := getHostPathVolumesForTheControlPlane(rt.cfg)
// Avoid unit test errors when the flexvolume is mounted
if _, ok := mounts.volumes[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {
delete(mounts.volumes[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName)
}
if _, ok := mounts.volumeMounts[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {
delete(mounts.volumeMounts[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName)
}
if !reflect.DeepEqual(mounts.volumes, rt.vol) {
t.Errorf(
"failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v",

2
cmd/kubeadm/app/phases/upgrade/compute_test.go
View File

@@ -430,7 +430,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.0-alpha.2",
KubeadmVersion: "v1.10.0-alpha.2",
DNSVersion: "1.14.7",
EtcdVersion: "3.1.11",
EtcdVersion: "3.1.10",
},
},
},

26
cmd/kubeadm/app/preflight/checks.go
View File

@@ -75,7 +75,7 @@ type Error struct {
}
func (e *Error) Error() string {
return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-checks-errors=...`")
return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`")
}
// Checker validates the state of the system to ensure kubeadm will be
@@ -837,9 +837,9 @@ func getEtcdVersionResponse(client *http.Client, url string, target interface{})
}
// RunInitMasterChecks executes all individual, applicable to Master node checks.
func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfiguration, criSocket string, ignoreChecksErrors sets.String) error {
func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfiguration, criSocket string, ignorePreflightErrors sets.String) error {
// First, check if we're root separately from the other preflight checks and fail fast
if err := RunRootCheckOnly(ignoreChecksErrors); err != nil {
if err := RunRootCheckOnly(ignorePreflightErrors); err != nil {
return err
}
@@ -929,13 +929,13 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfi
)
}
}
return RunChecks(checks, os.Stderr, ignoreChecksErrors)
return RunChecks(checks, os.Stderr, ignorePreflightErrors)
}
// RunJoinNodeChecks executes all individual, applicable to node checks.
func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.NodeConfiguration, criSocket string, ignoreChecksErrors sets.String) error {
func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.NodeConfiguration, criSocket string, ignorePreflightErrors sets.String) error {
// First, check if we're root separately from the other preflight checks and fail fast
if err := RunRootCheckOnly(ignoreChecksErrors); err != nil {
if err := RunRootCheckOnly(ignorePreflightErrors); err != nil {
return err
}
@@ -987,21 +987,21 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.NodeConfigura
}
}
}
return RunChecks(checks, os.Stderr, ignoreChecksErrors)
return RunChecks(checks, os.Stderr, ignorePreflightErrors)
}
// RunRootCheckOnly initializes checks slice of structs and call RunChecks
func RunRootCheckOnly(ignoreChecksErrors sets.String) error {
func RunRootCheckOnly(ignorePreflightErrors sets.String) error {
checks := []Checker{
IsPrivilegedUserCheck{},
}
return RunChecks(checks, os.Stderr, ignoreChecksErrors)
return RunChecks(checks, os.Stderr, ignorePreflightErrors)
}
// RunChecks runs each check, displays it's warnings/errors, and once all
// are processed will exit if any errors occurred.
func RunChecks(checks []Checker, ww io.Writer, ignoreChecksErrors sets.String) error {
func RunChecks(checks []Checker, ww io.Writer, ignorePreflightErrors sets.String) error {
type checkErrors struct {
Name string
Errors []error
@@ -1012,7 +1012,7 @@ func RunChecks(checks []Checker, ww io.Writer, ignoreChecksErrors sets.String) e
name := c.Name()
warnings, errs := c.Check()
if setHasItemOrAll(ignoreChecksErrors, name) {
if setHasItemOrAll(ignorePreflightErrors, name) {
// Decrease severity of errors to warnings for this check
warnings = append(warnings, errs...)
errs = []error{}
@@ -1038,8 +1038,8 @@ func RunChecks(checks []Checker, ww io.Writer, ignoreChecksErrors sets.String) e
}
// TryStartKubelet attempts to bring up kubelet service
func TryStartKubelet(ignoreChecksErrors sets.String) {
if setHasItemOrAll(ignoreChecksErrors, "StartKubelet") {
func TryStartKubelet(ignorePreflightErrors sets.String) {
if setHasItemOrAll(ignorePreflightErrors, "StartKubelet") {
return
}
// If we notice that the kubelet service is inactive, try to start it

27
cmd/kubeadm/app/preflight/checks_test.go
View File

@@ -619,3 +619,30 @@ func TestKubeletVersionCheck(t *testing.T) {
}
}
func TestSetHasItemOrAll(t *testing.T) {
var tests = []struct {
ignoreSet sets.String
testString string
expectedResult bool
}{
{sets.NewString(), "foo", false},
{sets.NewString("all"), "foo", true},
{sets.NewString("all", "bar"), "foo", true},
{sets.NewString("bar"), "foo", false},
{sets.NewString("baz", "foo", "bar"), "foo", true},
{sets.NewString("baz", "bar", "foo"), "Foo", true},
}
for _, rt := range tests {
result := setHasItemOrAll(rt.ignoreSet, rt.testString)
if result != rt.expectedResult {
t.Errorf(
"setHasItemOrAll: expected: %v actual: %v (arguments: %q, %q)",
rt.expectedResult, result,
rt.ignoreSet,
rt.testString,
)
}
}
}

41
docs/api-reference/extensions/v1beta1/definitions.html generated
View File

@@ -6728,6 +6728,40 @@ Both these may change in the future. Incoming requests are matched against the h
</tbody>
</table>
</div>
<div class="sect2">
<h3 id="_v1beta1_allowedflexvolume">v1beta1.AllowedFlexVolume</h3>
<div class="paragraph">
<p>AllowedFlexVolume represents a single Flexvolume that is allowed to be used.</p>
</div>
<table class="tableblock frame-all grid-all" style="width:100%; ">
<colgroup>
<col style="width:20%;">
<col style="width:20%;">
<col style="width:20%;">
<col style="width:20%;">
<col style="width:20%;">
</colgroup>
<thead>
<tr>
<th class="tableblock halign-left valign-top">Name</th>
<th class="tableblock halign-left valign-top">Description</th>
<th class="tableblock halign-left valign-top">Required</th>
<th class="tableblock halign-left valign-top">Schema</th>
<th class="tableblock halign-left valign-top">Default</th>
</tr>
</thead>
<tbody>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">driver</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Driver is the name of the Flexvolume driver.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
</div>
<div class="sect2">
<h3 id="_v1_apiresource">v1.APIResource</h3>
@@ -7972,6 +8006,13 @@ Both these may change in the future. Incoming requests are matched against the h
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1beta1_allowedhostpath">v1beta1.AllowedHostPath</a> array</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">allowedFlexVolumes</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the "Volumes" field.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock"><a href="#_v1beta1_allowedflexvolume">v1beta1.AllowedFlexVolume</a> array</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>

3
hack/lib/version.sh
View File

@@ -22,6 +22,7 @@
# source code.
# KUBE_GIT_TREE_STATE - "clean" indicates no changes since the git commit id
# "dirty" indicates source code changes after the git commit id
# "archive" indicates the tree was produced by 'git archive'
# KUBE_GIT_VERSION - "vX.Y" used to indicate the last release version.
# KUBE_GIT_MAJOR - The major part of the version
# KUBE_GIT_MINOR - The minor component of the version
@@ -40,7 +41,7 @@ kube::version::get_version_vars() {
# we likely don't have a git tree, but these magic values may be filled in.
if [[ '$Format:%%$' == "%" ]]; then
KUBE_GIT_COMMIT='$Format:%H$'
KUBE_GIT_TREE_STATE="git archive"
KUBE_GIT_TREE_STATE="archive"
# When a 'git archive' is exported, the '$Format:%D$' below will look
# something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: '
# can be extracted from it.

11
pkg/apis/extensions/types.go
View File

@@ -860,6 +860,11 @@ type PodSecurityPolicySpec struct {
// AllowedHostPaths is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
AllowedHostPaths []AllowedHostPath
// AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
// Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
// is allowed in the "Volumes" field.
// +optional
AllowedFlexVolumes []AllowedFlexVolume
}
// AllowedHostPath defines the host volume conditions that will be enabled by a policy
@@ -923,6 +928,12 @@ var (
All FSType = "*"
)
// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
type AllowedFlexVolume struct {
// Driver is the name of the Flexvolume driver.
Driver string
}
// SELinuxStrategyOptions defines the strategy type and any options used to create the strategy.
type SELinuxStrategyOptions struct {
// Rule is the strategy that will dictate the allowable labels that may be set.

24
pkg/apis/extensions/v1beta1/zz_generated.conversion.go generated
View File

@@ -41,6 +41,8 @@ func init() {
// Public to allow building arbitrary schemes.
func RegisterConversions(scheme *runtime.Scheme) error {
return scheme.AddGeneratedConversionFuncs(
Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume,
Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume,
Convert_v1beta1_AllowedHostPath_To_extensions_AllowedHostPath,
Convert_extensions_AllowedHostPath_To_v1beta1_AllowedHostPath,
Convert_v1beta1_CustomMetricCurrentStatus_To_extensions_CustomMetricCurrentStatus,
@@ -140,6 +142,26 @@ func RegisterConversions(scheme *runtime.Scheme) error {
)
}
func autoConvert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in *v1beta1.AllowedFlexVolume, out *extensions.AllowedFlexVolume, s conversion.Scope) error {
out.Driver = in.Driver
return nil
}
// Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume is an autogenerated conversion function.
func Convert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in *v1beta1.AllowedFlexVolume, out *extensions.AllowedFlexVolume, s conversion.Scope) error {
return autoConvert_v1beta1_AllowedFlexVolume_To_extensions_AllowedFlexVolume(in, out, s)
}
func autoConvert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in *extensions.AllowedFlexVolume, out *v1beta1.AllowedFlexVolume, s conversion.Scope) error {
out.Driver = in.Driver
return nil
}
// Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume is an autogenerated conversion function.
func Convert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in *extensions.AllowedFlexVolume, out *v1beta1.AllowedFlexVolume, s conversion.Scope) error {
return autoConvert_extensions_AllowedFlexVolume_To_v1beta1_AllowedFlexVolume(in, out, s)
}
func autoConvert_v1beta1_AllowedHostPath_To_extensions_AllowedHostPath(in *v1beta1.AllowedHostPath, out *extensions.AllowedHostPath, s conversion.Scope) error {
out.PathPrefix = in.PathPrefix
return nil
@@ -1072,6 +1094,7 @@ func autoConvert_v1beta1_PodSecurityPolicySpec_To_extensions_PodSecurityPolicySp
return err
}
out.AllowedHostPaths = *(*[]extensions.AllowedHostPath)(unsafe.Pointer(&in.AllowedHostPaths))
out.AllowedFlexVolumes = *(*[]extensions.AllowedFlexVolume)(unsafe.Pointer(&in.AllowedFlexVolumes))
return nil
}
@@ -1108,6 +1131,7 @@ func autoConvert_extensions_PodSecurityPolicySpec_To_v1beta1_PodSecurityPolicySp
return err
}
out.AllowedHostPaths = *(*[]v1beta1.AllowedHostPath)(unsafe.Pointer(&in.AllowedHostPaths))
out.AllowedFlexVolumes = *(*[]v1beta1.AllowedFlexVolume)(unsafe.Pointer(&in.AllowedFlexVolumes))
return nil
}

16
pkg/apis/extensions/validation/validation.go
View File

@@ -655,6 +655,7 @@ func ValidatePodSecurityPolicySpec(spec *extensions.PodSecurityPolicySpec, fldPa
allErrs = append(allErrs, validatePSPCapsAgainstDrops(spec.RequiredDropCapabilities, spec.AllowedCapabilities, field.NewPath("allowedCapabilities"))...)
allErrs = append(allErrs, validatePSPDefaultAllowPrivilegeEscalation(fldPath.Child("defaultAllowPrivilegeEscalation"), spec.DefaultAllowPrivilegeEscalation, spec.AllowPrivilegeEscalation)...)
allErrs = append(allErrs, validatePSPAllowedHostPaths(fldPath.Child("allowedHostPaths"), spec.AllowedHostPaths)...)
allErrs = append(allErrs, validatePSPAllowedFlexVolumes(fldPath.Child("allowedFlexVolumes"), spec.AllowedFlexVolumes)...)
return allErrs
}
@@ -721,6 +722,20 @@ func validatePSPAllowedHostPaths(fldPath *field.Path, allowedHostPaths []extensi
return allErrs
}
// validatePSPAllowedFlexVolumes
func validatePSPAllowedFlexVolumes(fldPath *field.Path, flexVolumes []extensions.AllowedFlexVolume) field.ErrorList {
allErrs := field.ErrorList{}
if len(flexVolumes) > 0 {
for idx, fv := range flexVolumes {
if len(fv.Driver) == 0 {
allErrs = append(allErrs, field.Required(fldPath.Child("allowedFlexVolumes").Index(idx).Child("driver"),
"must specify a driver"))
}
}
}
return allErrs
}
// validatePSPSELinux validates the SELinux fields of PodSecurityPolicy.
func validatePSPSELinux(fldPath *field.Path, seLinux *extensions.SELinuxStrategyOptions) field.ErrorList {
allErrs := field.ErrorList{}
@@ -802,7 +817,6 @@ func validatePodSecurityPolicyVolumes(fldPath *field.Path, volumes []extensions.
allErrs = append(allErrs, field.NotSupported(fldPath.Child("volumes"), v, allowed.List()))
}
}
return allErrs
}

29
pkg/apis/extensions/validation/validation_test.go
View File

@@ -2450,6 +2450,13 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
pe := true
invalidDefaultAllowPrivilegeEscalation.Spec.DefaultAllowPrivilegeEscalation = &pe
emptyFlexDriver := validPSP()
emptyFlexDriver.Spec.Volumes = []extensions.FSType{extensions.FlexVolume}
emptyFlexDriver.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{{}}
nonEmptyFlexVolumes := validPSP()
nonEmptyFlexVolumes.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{{Driver: "example/driver"}}
type testCase struct {
psp *extensions.PodSecurityPolicy
errorType field.ErrorType
@@ -2581,6 +2588,11 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
errorType: field.ErrorTypeInvalid,
errorDetail: "must not contain '..'",
},
"empty flex volume driver": {
psp: emptyFlexDriver,
errorType: field.ErrorTypeRequired,
errorDetail: "must specify a driver",
},
}
for k, v := range errorCases {
@@ -2660,6 +2672,17 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
validDefaultAllowPrivilegeEscalation.Spec.DefaultAllowPrivilegeEscalation = &pe
validDefaultAllowPrivilegeEscalation.Spec.AllowPrivilegeEscalation = true
flexvolumeWhenFlexVolumesAllowed := validPSP()
flexvolumeWhenFlexVolumesAllowed.Spec.Volumes = []extensions.FSType{extensions.FlexVolume}
flexvolumeWhenFlexVolumesAllowed.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{
{Driver: "example/driver1"},
}
flexvolumeWhenAllVolumesAllowed := validPSP()
flexvolumeWhenAllVolumesAllowed.Spec.Volumes = []extensions.FSType{extensions.All}
flexvolumeWhenAllVolumesAllowed.Spec.AllowedFlexVolumes = []extensions.AllowedFlexVolume{
{Driver: "example/driver2"},
}
successCases := map[string]struct {
psp *extensions.PodSecurityPolicy
}{
@@ -2690,6 +2713,12 @@ func TestValidatePodSecurityPolicy(t *testing.T) {
"valid defaultAllowPrivilegeEscalation as true": {
psp: validDefaultAllowPrivilegeEscalation,
},
"allow white-listed flexVolume when flex volumes are allowed": {
psp: flexvolumeWhenFlexVolumesAllowed,
},
"allow white-listed flexVolume when all volumes are allowed": {
psp: flexvolumeWhenAllVolumesAllowed,
},
}
for k, v := range successCases {

21
pkg/apis/extensions/zz_generated.deepcopy.go generated
View File

@@ -26,6 +26,22 @@ import (
core "k8s.io/kubernetes/pkg/apis/core"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedFlexVolume) DeepCopyInto(out *AllowedFlexVolume) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedFlexVolume.
func (in *AllowedFlexVolume) DeepCopy() *AllowedFlexVolume {
if in == nil {
return nil
}
out := new(AllowedFlexVolume)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedHostPath) DeepCopyInto(out *AllowedHostPath) {
*out = *in
@@ -919,6 +935,11 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
*out = make([]AllowedHostPath, len(*in))
copy(*out, *in)
}
if in.AllowedFlexVolumes != nil {
in, out := &in.AllowedFlexVolumes, &out.AllowedFlexVolumes
*out = make([]AllowedFlexVolume, len(*in))
copy(*out, *in)
}
return
}

1
pkg/cloudprovider/providers/aws/BUILD
View File

@@ -50,6 +50,7 @@ go_library(
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

128
pkg/cloudprovider/providers/aws/aws.go
View File

@@ -48,6 +48,7 @@ import (
"path"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
@@ -285,6 +286,10 @@ type EC2 interface {
// Delete an EBS volume
DeleteVolume(*ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error)
ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error)
DescribeVolumeModifications(*ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error)
DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error)
CreateSecurityGroup(*ec2.CreateSecurityGroupInput) (*ec2.CreateSecurityGroupOutput, error)
@@ -453,6 +458,9 @@ type Volumes interface {
// Check if disks specified in argument map are still attached to their respective nodes.
DisksAreAttached(map[types.NodeName][]KubernetesVolumeID) (map[types.NodeName]map[KubernetesVolumeID]bool, error)
// Expand the disk to new size
ResizeDisk(diskName KubernetesVolumeID, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error)
}
// InstanceGroups is an interface for managing cloud-managed instance groups / autoscaling instance groups
@@ -833,6 +841,36 @@ func (s *awsSdkEC2) DeleteVolume(request *ec2.DeleteVolumeInput) (*ec2.DeleteVol
return resp, err
}
func (s *awsSdkEC2) ModifyVolume(request *ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) {
requestTime := time.Now()
resp, err := s.ec2.ModifyVolume(request)
timeTaken := time.Since(requestTime).Seconds()
recordAwsMetric("modify_volume", timeTaken, err)
return resp, err
}
func (s *awsSdkEC2) DescribeVolumeModifications(request *ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error) {
requestTime := time.Now()
results := []*ec2.VolumeModification{}
var nextToken *string
for {
resp, err := s.ec2.DescribeVolumesModifications(request)
if err != nil {
recordAwsMetric("describe_volume_modification", 0, err)
return nil, fmt.Errorf("error listing volume modifictions : %v", err)
}
results = append(results, resp.VolumesModifications...)
nextToken = resp.NextToken
if aws.StringValue(nextToken) == "" {
break
}
request.NextToken = nextToken
}
timeTaken := time.Since(requestTime).Seconds()
recordAwsMetric("describe_volume_modification", timeTaken, nil)
return results, nil
}
func (s *awsSdkEC2) DescribeSubnets(request *ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) {
// Subnets are not paged
response, err := s.ec2.DescribeSubnets(request)
@@ -1653,6 +1691,65 @@ func (d *awsDisk) describeVolume() (*ec2.Volume, error) {
return volumes[0], nil
}
func (d *awsDisk) describeVolumeModification() (*ec2.VolumeModification, error) {
volumeID := d.awsID
request := &ec2.DescribeVolumesModificationsInput{
VolumeIds: []*string{volumeID.awsString()},
}
volumeMods, err := d.ec2.DescribeVolumeModifications(request)
if err != nil {
return nil, fmt.Errorf("error describing volume modification %s with %v", volumeID, err)
}
if len(volumeMods) == 0 {
return nil, fmt.Errorf("no volume modifications found for %s", volumeID)
}
lastIndex := len(volumeMods) - 1
return volumeMods[lastIndex], nil
}
func (d *awsDisk) modifyVolume(requestGiB int64) (int64, error) {
volumeID := d.awsID
request := &ec2.ModifyVolumeInput{
VolumeId: volumeID.awsString(),
Size: aws.Int64(requestGiB),
}
output, err := d.ec2.ModifyVolume(request)
if err != nil {
modifyError := fmt.Errorf("AWS modifyVolume failed for %s with %v", volumeID, err)
return requestGiB, modifyError
}
volumeModification := output.VolumeModification
if aws.StringValue(volumeModification.ModificationState) == ec2.VolumeModificationStateCompleted {
return aws.Int64Value(volumeModification.TargetSize), nil
}
backoff := wait.Backoff{
Duration: 1 * time.Second,
Factor: 2,
Steps: 10,
}
checkForResize := func() (bool, error) {
volumeModification, err := d.describeVolumeModification()
if err != nil {
return false, err
}
if aws.StringValue(volumeModification.ModificationState) == ec2.VolumeModificationStateCompleted {
return true, nil
}
return false, nil
}
waitWithErr := wait.ExponentialBackoff(backoff, checkForResize)
return requestGiB, waitWithErr
}
// applyUnSchedulableTaint applies a unschedulable taint to a node after verifying
// if node has become unusable because of volumes getting stuck in attaching state.
func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) {
@@ -2321,6 +2418,37 @@ func (c *Cloud) DisksAreAttached(nodeDisks map[types.NodeName][]KubernetesVolume
return attached, nil
}
func (c *Cloud) ResizeDisk(
diskName KubernetesVolumeID,
oldSize resource.Quantity,
newSize resource.Quantity) (resource.Quantity, error) {
awsDisk, err := newAWSDisk(c, diskName)
if err != nil {
return oldSize, err
}
volumeInfo, err := awsDisk.describeVolume()
if err != nil {
descErr := fmt.Errorf("AWS.ResizeDisk Error describing volume %s with %v", diskName, err)
return oldSize, descErr
}
requestBytes := newSize.Value()
// AWS resizes in chunks of GiB (not GB)
requestGiB := volume.RoundUpSize(requestBytes, 1024*1024*1024)
newSizeQuant := resource.MustParse(fmt.Sprintf("%dGi", requestGiB))
// If disk already if of greater or equal size than requested we return
if aws.Int64Value(volumeInfo.Size) >= requestGiB {
return newSizeQuant, nil
}
_, err = awsDisk.modifyVolume(requestGiB)
if err != nil {
return oldSize, err
}
return newSizeQuant, nil
}
// Gets the current load balancer state
func (c *Cloud) describeLoadBalancer(name string) (*elb.LoadBalancerDescription, error) {
request := &elb.DescribeLoadBalancersInput{}

8
pkg/cloudprovider/providers/aws/aws_fakes.go
View File

@@ -203,6 +203,14 @@ func (ec2i *FakeEC2Impl) RevokeSecurityGroupIngress(*ec2.RevokeSecurityGroupIngr
panic("Not implemented")
}
func (ec2i *FakeEC2Impl) DescribeVolumeModifications(*ec2.DescribeVolumesModificationsInput) ([]*ec2.VolumeModification, error) {
panic("Not implemented")
}
func (ec2i *FakeEC2Impl) ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) {
panic("Not implemented")
}
func (ec2i *FakeEC2Impl) CreateSubnet(request *ec2.Subnet) (*ec2.CreateSubnetOutput, error) {
ec2i.Subnets = append(ec2i.Subnets, request)
response := &ec2.CreateSubnetOutput{

1
pkg/controller/BUILD
View File

@@ -133,6 +133,7 @@ filegroup(
"//pkg/controller/volume/events:all-srcs",
"//pkg/controller/volume/expand:all-srcs",
"//pkg/controller/volume/persistentvolume:all-srcs",
"//pkg/controller/volume/pvcprotection:all-srcs",
],
tags = ["automanaged"],
)

6
pkg/controller/node/ipam/BUILD
View File

@@ -16,12 +16,15 @@ go_test(
importpath = "k8s.io/kubernetes/pkg/controller/node/ipam",
library = ":go_default_library",
deps = [
"//pkg/controller:go_default_library",
"//pkg/controller/node/ipam/cidrset:go_default_library",
"//pkg/controller/node/ipam/test:go_default_library",
"//pkg/controller/testutil:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)
@@ -41,6 +44,7 @@ go_library(
deps = [
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/controller/node/ipam/cidrset:go_default_library",
"//pkg/controller/node/ipam/sync:go_default_library",
"//pkg/controller/node/util:go_default_library",
@@ -52,12 +56,14 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/scheme:go_default_library",

10
pkg/controller/node/ipam/cidr_allocator.go
View File

@@ -80,12 +80,12 @@ type CIDRAllocator interface {
AllocateOrOccupyCIDR(node *v1.Node) error
// ReleaseCIDR releases the CIDR of the removed node
ReleaseCIDR(node *v1.Node) error
// Register allocator with the nodeInformer for updates.
Register(nodeInformer informers.NodeInformer)
// Run starts all the working logic of the allocator.
Run(stopCh <-chan struct{})
}
// New creates a new CIDR range allocator.
func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, allocatorType CIDRAllocatorType, clusterCIDR, serviceCIDR *net.IPNet, nodeCIDRMaskSize int) (CIDRAllocator, error) {
func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer, allocatorType CIDRAllocatorType, clusterCIDR, serviceCIDR *net.IPNet, nodeCIDRMaskSize int) (CIDRAllocator, error) {
nodeList, err := listNodes(kubeClient)
if err != nil {
return nil, err
@@ -93,9 +93,9 @@ func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, allocato
switch allocatorType {
case RangeAllocatorType:
return NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList)
return NewCIDRRangeAllocator(kubeClient, nodeInformer, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList)
case CloudAllocatorType:
return NewCloudCIDRAllocator(kubeClient, cloud)
return NewCloudCIDRAllocator(kubeClient, cloud, nodeInformer)
default:
return nil, fmt.Errorf("Invalid CIDR allocator type: %v", allocatorType)
}

60
pkg/controller/node/ipam/cloud_cidr_allocator.go
View File

@@ -25,9 +25,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
informers "k8s.io/client-go/informers/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
@@ -37,6 +38,7 @@ import (
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/node/util"
nodeutil "k8s.io/kubernetes/pkg/util/node"
)
@@ -49,6 +51,12 @@ type cloudCIDRAllocator struct {
client clientset.Interface
cloud *gce.GCECloud
// nodeLister is able to list/get nodes and is populated by the shared informer passed to
// NewCloudCIDRAllocator.
nodeLister corelisters.NodeLister
// nodesSynced returns true if the node shared informer has been synced at least once.
nodesSynced cache.InformerSynced
// Channel that is used to pass updating Nodes to the background.
// This increases the throughput of CIDR assignment by parallelization
// and not blocking on long operations (which shouldn't be done from
@@ -64,7 +72,7 @@ type cloudCIDRAllocator struct {
var _ CIDRAllocator = (*cloudCIDRAllocator)(nil)
// NewCloudCIDRAllocator creates a new cloud CIDR allocator.
func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Interface) (CIDRAllocator, error) {
func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer) (CIDRAllocator, error) {
if client == nil {
glog.Fatalf("kubeClient is nil when starting NodeController")
}
@@ -84,20 +92,45 @@ func NewCloudCIDRAllocator(client clientset.Interface, cloud cloudprovider.Inter
ca := &cloudCIDRAllocator{
client: client,
cloud: gceCloud,
nodeLister: nodeInformer.Lister(),
nodesSynced: nodeInformer.Informer().HasSynced,
nodeUpdateChannel: make(chan string, cidrUpdateQueueSize),
recorder: recorder,
nodesInProcessing: sets.NewString(),
}
for i := 0; i < cidrUpdateWorkers; i++ {
// TODO: Take stopChan as an argument to NewCloudCIDRAllocator and pass it to the worker.
go ca.worker(wait.NeverStop)
}
nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: util.CreateAddNodeHandler(ca.AllocateOrOccupyCIDR),
UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
if newNode.Spec.PodCIDR == "" {
return ca.AllocateOrOccupyCIDR(newNode)
}
return nil
}),
DeleteFunc: util.CreateDeleteNodeHandler(ca.ReleaseCIDR),
})
glog.V(0).Infof("Using cloud CIDR allocator (provider: %v)", cloud.ProviderName())
return ca, nil
}
func (ca *cloudCIDRAllocator) Run(stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
glog.Infof("Starting cloud CIDR allocator")
defer glog.Infof("Shutting down cloud CIDR allocator")
if !controller.WaitForCacheSync("cidrallocator", stopCh, ca.nodesSynced) {
return
}
for i := 0; i < cidrUpdateWorkers; i++ {
go ca.worker(stopCh)
}
<-stopCh
}
func (ca *cloudCIDRAllocator) worker(stopChan <-chan struct{}) {
for {
select {
@@ -169,7 +202,7 @@ func (ca *cloudCIDRAllocator) updateCIDRAllocation(nodeName string) error {
for rep := 0; rep < cidrUpdateRetries; rep++ {
// TODO: change it to using PATCH instead of full Node updates.
node, err = ca.client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
node, err = ca.nodeLister.Get(nodeName)
if err != nil {
glog.Errorf("Failed while getting node %v to retry updating Node.Spec.PodCIDR: %v", nodeName, err)
continue
@@ -218,16 +251,3 @@ func (ca *cloudCIDRAllocator) ReleaseCIDR(node *v1.Node) error {
node.Name, node.Spec.PodCIDR)
return nil
}
func (ca *cloudCIDRAllocator) Register(nodeInformer informers.NodeInformer) {
nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: util.CreateAddNodeHandler(ca.AllocateOrOccupyCIDR),
UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
if newNode.Spec.PodCIDR == "" {
return ca.AllocateOrOccupyCIDR(newNode)
}
return nil
}),
DeleteFunc: util.CreateDeleteNodeHandler(ca.ReleaseCIDR),
})
}

101
pkg/controller/node/ipam/range_allocator.go
View File

@@ -25,16 +25,16 @@ import (
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
informers "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/node/ipam/cidrset"
"k8s.io/kubernetes/pkg/controller/node/util"
)
@@ -45,6 +45,12 @@ type rangeAllocator struct {
clusterCIDR *net.IPNet
maxCIDRs int
// nodeLister is able to list/get nodes and is populated by the shared informer passed to
// NewCloudCIDRAllocator.
nodeLister corelisters.NodeLister
// nodesSynced returns true if the node shared informer has been synced at least once.
nodesSynced cache.InformerSynced
// Channel that is used to pass updating Nodes with assigned CIDRs to the background
// This increases a throughput of CIDR assignment by not blocking on long operations.
nodeCIDRUpdateChannel chan nodeAndCIDR
@@ -59,7 +65,7 @@ type rangeAllocator struct {
// Caller must ensure subNetMaskSize is not less than cluster CIDR mask size.
// Caller must always pass in a list of existing nodes so the new allocator
// can initialize its CIDR map. NodeList is only nil in testing.
func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, subNetMaskSize int, nodeList *v1.NodeList) (CIDRAllocator, error) {
func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.NodeInformer, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, subNetMaskSize int, nodeList *v1.NodeList) (CIDRAllocator, error) {
if client == nil {
glog.Fatalf("kubeClient is nil when starting NodeController")
}
@@ -78,6 +84,8 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s
client: client,
cidrs: set,
clusterCIDR: clusterCIDR,
nodeLister: nodeInformer.Lister(),
nodesSynced: nodeInformer.Informer().HasSynced,
nodeCIDRUpdateChannel: make(chan nodeAndCIDR, cidrUpdateQueueSize),
recorder: recorder,
nodesInProcessing: sets.NewString(),
@@ -107,14 +115,57 @@ func NewCIDRRangeAllocator(client clientset.Interface, clusterCIDR *net.IPNet, s
}
}
}
for i := 0; i < cidrUpdateWorkers; i++ {
// TODO: Take stopChan as an argument to NewCIDRRangeAllocator and pass it to the worker.
go ra.worker(wait.NeverStop)
}
nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: util.CreateAddNodeHandler(ra.AllocateOrOccupyCIDR),
UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
// If the PodCIDR is not empty we either:
// - already processed a Node that already had a CIDR after NC restarted
// (cidr is marked as used),
// - already processed a Node successfully and allocated a CIDR for it
// (cidr is marked as used),
// - already processed a Node but we did saw a "timeout" response and
// request eventually got through in this case we haven't released
// the allocated CIDR (cidr is still marked as used).
// There's a possible error here:
// - NC sees a new Node and assigns a CIDR X to it,
// - Update Node call fails with a timeout,
// - Node is updated by some other component, NC sees an update and
// assigns CIDR Y to the Node,
// - Both CIDR X and CIDR Y are marked as used in the local cache,
// even though Node sees only CIDR Y
// The problem here is that in in-memory cache we see CIDR X as marked,
// which prevents it from being assigned to any new node. The cluster
// state is correct.
// Restart of NC fixes the issue.
if newNode.Spec.PodCIDR == "" {
return ra.AllocateOrOccupyCIDR(newNode)
}
return nil
}),
DeleteFunc: util.CreateDeleteNodeHandler(ra.ReleaseCIDR),
})
return ra, nil
}
func (r *rangeAllocator) Run(stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
glog.Infof("Starting range CIDR allocator")
defer glog.Infof("Shutting down range CIDR allocator")
if !controller.WaitForCacheSync("cidrallocator", stopCh, r.nodesSynced) {
return
}
for i := 0; i < cidrUpdateWorkers; i++ {
go r.worker(stopCh)
}
<-stopCh
}
func (r *rangeAllocator) worker(stopChan <-chan struct{}) {
for {
select {
@@ -232,7 +283,7 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
podCIDR := data.cidr.String()
for rep := 0; rep < cidrUpdateRetries; rep++ {
// TODO: change it to using PATCH instead of full Node updates.
node, err = r.client.CoreV1().Nodes().Get(data.nodeName, metav1.GetOptions{})
node, err = r.nodeLister.Get(data.nodeName)
if err != nil {
glog.Errorf("Failed while getting node %v to retry updating Node.Spec.PodCIDR: %v", data.nodeName, err)
continue
@@ -269,35 +320,3 @@ func (r *rangeAllocator) updateCIDRAllocation(data nodeAndCIDR) error {
}
return err
}
func (r *rangeAllocator) Register(nodeInformer informers.NodeInformer) {
nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: util.CreateAddNodeHandler(r.AllocateOrOccupyCIDR),
UpdateFunc: util.CreateUpdateNodeHandler(func(_, newNode *v1.Node) error {
// If the PodCIDR is not empty we either:
// - already processed a Node that already had a CIDR after NC restarted
// (cidr is marked as used),
// - already processed a Node successfully and allocated a CIDR for it
// (cidr is marked as used),
// - already processed a Node but we did saw a "timeout" response and
// request eventually got through in this case we haven't released
// the allocated CIDR (cidr is still marked as used).
// There's a possible error here:
// - NC sees a new Node and assigns a CIDR X to it,
// - Update Node call fails with a timeout,
// - Node is updated by some other component, NC sees an update and
// assigns CIDR Y to the Node,
// - Both CIDR X and CIDR Y are marked as used in the local cache,
// even though Node sees only CIDR Y
// The problem here is that in in-memory cache we see CIDR X as marked,
// which prevents it from being assigned to any new node. The cluster
// state is correct.
// Restart of NC fixes the issue.
if newNode.Spec.PodCIDR == "" {
return r.AllocateOrOccupyCIDR(newNode)
}
return nil
}),
DeleteFunc: util.CreateDeleteNodeHandler(r.ReleaseCIDR),
})
}

72
pkg/controller/node/ipam/range_allocator_test.go
View File

@@ -24,7 +24,10 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers"
coreinformers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/testutil"
)
@@ -32,6 +35,8 @@ const (
nodePollInterval = 100 * time.Millisecond
)
var alwaysReady = func() bool { return true }
func waitForUpdatedNodeWithTimeout(nodeHandler *testutil.FakeNodeHandler, number int, timeout time.Duration) error {
return wait.Poll(nodePollInterval, timeout, func() (bool, error) {
if len(nodeHandler.GetUpdatedNodesCopy()) >= number {
@@ -41,6 +46,19 @@ func waitForUpdatedNodeWithTimeout(nodeHandler *testutil.FakeNodeHandler, number
})
}
// Creates a fakeNodeInformer using the provided fakeNodeHandler.
func getFakeNodeInformer(fakeNodeHandler *testutil.FakeNodeHandler) coreinformers.NodeInformer {
fakeClient := &fake.Clientset{}
fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, controller.NoResyncPeriodFunc())
fakeNodeInformer := fakeInformerFactory.Core().V1().Nodes()
for _, node := range fakeNodeHandler.Existing {
fakeNodeInformer.Informer().GetStore().Add(node)
}
return fakeNodeInformer
}
func TestAllocateOrOccupyCIDRSuccess(t *testing.T) {
testCases := []struct {
description string
@@ -130,19 +148,23 @@ func TestAllocateOrOccupyCIDRSuccess(t *testing.T) {
expectedAllocatedCIDR string
allocatedCIDRs []string
}) {
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
// Initialize the range allocator.
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.nodesSynced = alwaysReady
rangeAllocator.recorder = testutil.NewFakeRecorder()
go allocator.Run(wait.NeverStop)
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.recorder = testutil.NewFakeRecorder()
if err = rangeAllocator.cidrs.Occupy(cidr); err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
}
@@ -212,19 +234,23 @@ func TestAllocateOrOccupyCIDRFailure(t *testing.T) {
subNetMaskSize int
allocatedCIDRs []string
}) {
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
// Initialize the range allocator.
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.nodesSynced = alwaysReady
rangeAllocator.recorder = testutil.NewFakeRecorder()
go allocator.Run(wait.NeverStop)
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)
@@ -324,19 +350,23 @@ func TestReleaseCIDRSuccess(t *testing.T) {
allocatedCIDRs []string
cidrsToRelease []string
}) {
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
// Initialize the range allocator.
allocator, _ := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.clusterCIDR, tc.serviceCIDR, tc.subNetMaskSize, nil)
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.nodesSynced = alwaysReady
rangeAllocator.recorder = testutil.NewFakeRecorder()
go allocator.Run(wait.NeverStop)
// this is a bit of white box testing
for _, allocated := range tc.allocatedCIDRs {
_, cidr, err := net.ParseCIDR(allocated)
if err != nil {
t.Fatalf("%v: unexpected error when parsing CIDR %v: %v", tc.description, allocated, err)
}
rangeAllocator, ok := allocator.(*rangeAllocator)
if !ok {
t.Logf("%v: found non-default implementation of CIDRAllocator, skipping white-box test...", tc.description)
return
}
rangeAllocator.recorder = testutil.NewFakeRecorder()
err = rangeAllocator.cidrs.Occupy(cidr)
if err != nil {
t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err)

9
pkg/controller/node/node_controller.go
View File

@@ -360,11 +360,10 @@ func NewNodeController(
} else {
var err error
nc.cidrAllocator, err = ipam.New(
kubeClient, cloud, nc.allocatorType, nc.clusterCIDR, nc.serviceCIDR, nodeCIDRMaskSize)
kubeClient, cloud, nodeInformer, nc.allocatorType, nc.clusterCIDR, nc.serviceCIDR, nodeCIDRMaskSize)
if err != nil {
return nil, err
}
nc.cidrAllocator.Register(nodeInformer)
}
}
@@ -585,6 +584,12 @@ func (nc *Controller) Run(stopCh <-chan struct{}) {
go wait.Until(nc.doEvictionPass, scheduler.NodeEvictionPeriod, wait.NeverStop)
}
if nc.allocateNodeCIDRs {
if nc.allocatorType != ipam.IPAMFromClusterAllocatorType && nc.allocatorType != ipam.IPAMFromCloudAllocatorType {
go nc.cidrAllocator.Run(wait.NeverStop)
}
}
<-stopCh
}

61
pkg/controller/volume/pvcprotection/BUILD Normal file
View File

@@ -0,0 +1,61 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["pvc_protection_controller.go"],
importpath = "k8s.io/kubernetes/pkg/controller/volume/pvcprotection",
visibility = ["//visibility:public"],
deps = [
"//pkg/controller:go_default_library",
"//pkg/util/metrics:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/volumehelper:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/util/workqueue:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["pvc_protection_controller_test.go"],
importpath = "k8s.io/kubernetes/pkg/controller/volume/pvcprotection",
library = ":go_default_library",
deps = [
"//pkg/controller:go_default_library",
"//pkg/volume/util:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

284
pkg/controller/volume/pvcprotection/pvc_protection_controller.go Normal file
View File

@@ -0,0 +1,284 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pvcprotection
import (
"fmt"
"time"
"github.com/golang/glog"
"k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
coreinformers "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/util/metrics"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
// Controller is controller that removes PVCProtectionFinalizer
// from PVCs that are used by no pods.
type Controller struct {
client clientset.Interface
pvcLister corelisters.PersistentVolumeClaimLister
pvcListerSynced cache.InformerSynced
podLister corelisters.PodLister
podListerSynced cache.InformerSynced
queue workqueue.RateLimitingInterface
}
// NewPVCProtectionController returns a new *{VCProtectionController.
func NewPVCProtectionController(pvcInformer coreinformers.PersistentVolumeClaimInformer, podInformer coreinformers.PodInformer, cl clientset.Interface) *Controller {
e := &Controller{
client: cl,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcprotection"),
}
if cl != nil && cl.CoreV1().RESTClient().GetRateLimiter() != nil {
metrics.RegisterMetricAndTrackRateLimiterUsage("persistentvolumeclaim_protection_controller", cl.CoreV1().RESTClient().GetRateLimiter())
}
e.pvcLister = pvcInformer.Lister()
e.pvcListerSynced = pvcInformer.Informer().HasSynced
pvcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: e.pvcAddedUpdated,
UpdateFunc: func(old, new interface{}) {
e.pvcAddedUpdated(new)
},
})
e.podLister = podInformer.Lister()
e.podListerSynced = podInformer.Informer().HasSynced
podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
e.podAddedDeletedUpdated(obj, false)
},
DeleteFunc: func(obj interface{}) {
e.podAddedDeletedUpdated(obj, true)
},
UpdateFunc: func(old, new interface{}) {
e.podAddedDeletedUpdated(new, false)
},
})
return e
}
// Run runs the controller goroutines.
func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
defer c.queue.ShutDown()
glog.Infof("Starting PVC protection controller")
defer glog.Infof("Shutting down PVC protection controller")
if !controller.WaitForCacheSync("PVC protection", stopCh, c.pvcListerSynced, c.podListerSynced) {
return
}
for i := 0; i < workers; i++ {
go wait.Until(c.runWorker, time.Second, stopCh)
}
<-stopCh
}
func (c *Controller) runWorker() {
for c.processNextWorkItem() {
}
}
// processNextWorkItem deals with one pvcKey off the queue. It returns false when it's time to quit.
func (c *Controller) processNextWorkItem() bool {
pvcKey, quit := c.queue.Get()
if quit {
return false
}
defer c.queue.Done(pvcKey)
pvcNamespace, pvcName, err := cache.SplitMetaNamespaceKey(pvcKey.(string))
if err != nil {
utilruntime.HandleError(fmt.Errorf("Error parsing PVC key %q: %v", pvcKey, err))
return true
}
err = c.processPVC(pvcNamespace, pvcName)
if err == nil {
c.queue.Forget(pvcKey)
return true
}
utilruntime.HandleError(fmt.Errorf("PVC %v failed with : %v", pvcKey, err))
c.queue.AddRateLimited(pvcKey)
return true
}
func (c *Controller) processPVC(pvcNamespace, pvcName string) error {
glog.V(4).Infof("Processing PVC %s/%s", pvcNamespace, pvcName)
startTime := time.Now()
defer func() {
glog.V(4).Infof("Finished processing PVC %s/%s (%v)", pvcNamespace, pvcName, time.Now().Sub(startTime))
}()
pvc, err := c.pvcLister.PersistentVolumeClaims(pvcNamespace).Get(pvcName)
if apierrs.IsNotFound(err) {
glog.V(4).Infof("PVC %s/%s not found, ignoring", pvcNamespace, pvcName)
return nil
}
if err != nil {
return err
}
if volumeutil.IsPVCBeingDeleted(pvc) && volumeutil.IsProtectionFinalizerPresent(pvc) {
// PVC should be deleted. Check if it's used and remove finalizer if
// it's not.
isUsed, err := c.isBeingUsed(pvc)
if err != nil {
return err
}
if !isUsed {
return c.removeFinalizer(pvc)
}
}
if !volumeutil.IsPVCBeingDeleted(pvc) && !volumeutil.IsProtectionFinalizerPresent(pvc) {
// PVC is not being deleted -> it should have the finalizer. The
// finalizer should be added by admission plugin, this is just to add
// the finalizer to old PVCs that were created before the admission
// plugin was enabled.
return c.addFinalizer(pvc)
}
return nil
}
func (c *Controller) addFinalizer(pvc *v1.PersistentVolumeClaim) error {
claimClone := pvc.DeepCopy()
volumeutil.AddProtectionFinalizer(claimClone)
_, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone)
if err != nil {
glog.V(3).Infof("Error adding protection finalizer to PVC %s/%s: %v", pvc.Namespace, pvc.Name)
return err
}
glog.V(3).Infof("Added protection finalizer to PVC %s/%s", pvc.Namespace, pvc.Name)
return nil
}
func (c *Controller) removeFinalizer(pvc *v1.PersistentVolumeClaim) error {
claimClone := pvc.DeepCopy()
volumeutil.RemoveProtectionFinalizer(claimClone)
_, err := c.client.CoreV1().PersistentVolumeClaims(claimClone.Namespace).Update(claimClone)
if err != nil {
glog.V(3).Infof("Error removing protection finalizer from PVC %s/%s: %v", pvc.Namespace, pvc.Name, err)
return err
}
glog.V(3).Infof("Removed protection finalizer from PVC %s/%s", pvc.Namespace, pvc.Name)
return nil
}
func (c *Controller) isBeingUsed(pvc *v1.PersistentVolumeClaim) (bool, error) {
pods, err := c.podLister.Pods(pvc.Namespace).List(labels.Everything())
if err != nil {
return false, err
}
for _, pod := range pods {
if pod.Spec.NodeName == "" {
// This pod is not scheduled. We have a predicated in scheduler that
// prevents scheduling pods with deletion timestamp, so we can be
// pretty sure it won't be scheduled in parallel to this check.
// Therefore this pod does not block the PVC from deletion.
glog.V(4).Infof("Skipping unscheduled pod %s when checking PVC %s/%s", pod.Name, pvc.Namespace, pvc.Name)
continue
}
if volumehelper.IsPodTerminated(pod, pod.Status) {
// This pod is being unmounted/detached or is already
// unmounted/detached. It does not block the PVC from deletion.
continue
}
for _, volume := range pod.Spec.Volumes {
if volume.PersistentVolumeClaim == nil {
continue
}
if volume.PersistentVolumeClaim.ClaimName == pvc.Name {
glog.V(2).Infof("Keeping PVC %s/%s, it is used by pod %s/%s", pvc.Namespace, pvc.Name, pod.Namespace, pod.Name)
return true, nil
}
}
}
glog.V(3).Infof("PVC %s/%s is unused", pvc.Namespace, pvc.Name)
return false, nil
}
// pvcAddedUpdated reacts to pvc added/updated/deleted events
func (c *Controller) pvcAddedUpdated(obj interface{}) {
pvc, ok := obj.(*v1.PersistentVolumeClaim)
if !ok {
utilruntime.HandleError(fmt.Errorf("PVC informer returned non-PVC object: %#v", obj))
return
}
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pvc)
if err != nil {
utilruntime.HandleError(fmt.Errorf("Couldn't get key for Persistent Volume Claim %#v: %v", pvc, err))
return
}
glog.V(4).Infof("Got event on PVC %s", key)
if (!volumeutil.IsPVCBeingDeleted(pvc) && !volumeutil.IsProtectionFinalizerPresent(pvc)) || (volumeutil.IsPVCBeingDeleted(pvc) && volumeutil.IsProtectionFinalizerPresent(pvc)) {
c.queue.Add(key)
}
}
// podAddedDeletedUpdated reacts to Pod events
func (c *Controller) podAddedDeletedUpdated(obj interface{}, deleted bool) {
pod, ok := obj.(*v1.Pod)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
utilruntime.HandleError(fmt.Errorf("Couldn't get object from tombstone %#v", obj))
return
}
pod, ok = tombstone.Obj.(*v1.Pod)
if !ok {
utilruntime.HandleError(fmt.Errorf("Tombstone contained object that is not a Pod %#v", obj))
return
}
}
// Filter out pods that can't help us to remove a finalizer on PVC
if !deleted && !volumehelper.IsPodTerminated(pod, pod.Status) && pod.Spec.NodeName != "" {
return
}
glog.V(4).Infof("Got event on pod %s/%s", pod.Namespace, pod.Name)
// Enqueue all PVCs that the pod uses
for _, volume := range pod.Spec.Volumes {
if volume.PersistentVolumeClaim != nil {
c.queue.Add(pod.Namespace + "/" + volume.PersistentVolumeClaim.ClaimName)
}
}
}

397
pkg/controller/volume/pvcprotection/pvc_protection_controller_test.go Normal file
View File

@@ -0,0 +1,397 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pvcprotection
import (
"errors"
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/golang/glog"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
clienttesting "k8s.io/client-go/testing"
"k8s.io/kubernetes/pkg/controller"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
)
type reaction struct {
verb string
resource string
reactorfn clienttesting.ReactionFunc
}
const (
defaultNS = "default"
defaultPVCName = "pvc1"
defaultPodName = "pod1"
defaultNodeName = "node1"
)
func pod() *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: defaultPodName,
Namespace: defaultNS,
},
Spec: v1.PodSpec{
NodeName: defaultNodeName,
},
Status: v1.PodStatus{
Phase: v1.PodPending,
},
}
}
func unscheduled(pod *v1.Pod) *v1.Pod {
pod.Spec.NodeName = ""
return pod
}
func withPVC(pvcName string, pod *v1.Pod) *v1.Pod {
volume := v1.Volume{
Name: pvcName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
},
},
}
pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
return pod
}
func withEmptyDir(pod *v1.Pod) *v1.Pod {
volume := v1.Volume{
Name: "emptyDir",
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}
pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
return pod
}
func withStatus(phase v1.PodPhase, pod *v1.Pod) *v1.Pod {
pod.Status.Phase = phase
return pod
}
func pvc() *v1.PersistentVolumeClaim {
return &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: defaultPVCName,
Namespace: defaultNS,
},
}
}
func withProtectionFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer)
return pvc
}
func deleted(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim {
pvc.DeletionTimestamp = &metav1.Time{}
return pvc
}
func generateUpdateErrorFunc(t *testing.T, failures int) clienttesting.ReactionFunc {
i := 0
return func(action clienttesting.Action) (bool, runtime.Object, error) {
i++
if i <= failures {
// Update fails
update, ok := action.(clienttesting.UpdateAction)
if !ok {
t.Fatalf("Reactor got non-update action: %+v", action)
}
acc, _ := meta.Accessor(update.GetObject())
return true, nil, apierrors.NewForbidden(update.GetResource().GroupResource(), acc.GetName(), errors.New("Mock error"))
}
// Update succeeds
return false, nil, nil
}
}
func TestPVCProtectionController(t *testing.T) {
pvcVer := schema.GroupVersionResource{
Group: v1.GroupName,
Version: "v1",
Resource: "persistentvolumeclaims",
}
tests := []struct {
name string
// Object to insert into fake kubeclient before the test starts.
initialObjects []runtime.Object
// Optional client reactors.
reactors []reaction
// PVC event to simulate. This PVC will be automatically added to
// initalObjects.
updatedPVC *v1.PersistentVolumeClaim
// Pod event to simulate. This Pod will be automatically added to
// initalObjects.
updatedPod *v1.Pod
// Pod event to similate. This Pod is *not* added to
// initalObjects.
deletedPod *v1.Pod
// List of expected kubeclient actions that should happen during the
// test.
expectedActions []clienttesting.Action
}{
//
// PVC events
//
{
name: "PVC without finalizer -> finalizer is added",
updatedPVC: pvc(),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
},
},
{
name: "PVC with finalizer -> no action",
updatedPVC: withProtectionFinalizer(pvc()),
expectedActions: []clienttesting.Action{},
},
{
name: "saving PVC finalizer fails -> controller retries",
updatedPVC: pvc(),
reactors: []reaction{
{
verb: "update",
resource: "persistentvolumeclaims",
reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/),
},
},
expectedActions: []clienttesting.Action{
// This fails
clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
// This fails too
clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
// This succeeds
clienttesting.NewUpdateAction(pvcVer, defaultNS, withProtectionFinalizer(pvc())),
},
},
{
name: "deleted PVC with finalizer -> finalizer is removed",
updatedPVC: deleted(withProtectionFinalizer(pvc())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
{
name: "finalizer removal fails -> controller retries",
updatedPVC: deleted(withProtectionFinalizer(pvc())),
reactors: []reaction{
{
verb: "update",
resource: "persistentvolumeclaims",
reactorfn: generateUpdateErrorFunc(t, 2 /* update fails twice*/),
},
},
expectedActions: []clienttesting.Action{
// Fails
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
// Fails too
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
// Succeeds
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
{
name: "deleted PVC with finalizer + pods with the PVC exists -> finalizer is not removed",
initialObjects: []runtime.Object{
withPVC(defaultPVCName, pod()),
},
updatedPVC: deleted(withProtectionFinalizer(pvc())),
expectedActions: []clienttesting.Action{},
},
{
name: "deleted PVC with finalizer + pods with unrelated PVC and EmptyDir exists -> finalizer is removed",
initialObjects: []runtime.Object{
withEmptyDir(withPVC("unrelatedPVC", pod())),
},
updatedPVC: deleted(withProtectionFinalizer(pvc())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
{
name: "deleted PVC with finalizer + pods with the PVC andis finished -> finalizer is removed",
initialObjects: []runtime.Object{
withStatus(v1.PodFailed, withPVC(defaultPVCName, pod())),
},
updatedPVC: deleted(withProtectionFinalizer(pvc())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
//
// Pod events
//
{
name: "updated running Pod -> no action",
initialObjects: []runtime.Object{
deleted(withProtectionFinalizer(pvc())),
},
updatedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())),
expectedActions: []clienttesting.Action{},
},
{
name: "updated finished Pod -> finalizer is removed",
initialObjects: []runtime.Object{
deleted(withProtectionFinalizer(pvc())),
},
updatedPod: withStatus(v1.PodSucceeded, withPVC(defaultPVCName, pod())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
{
name: "updated unscheduled Pod -> finalizer is removed",
initialObjects: []runtime.Object{
deleted(withProtectionFinalizer(pvc())),
},
updatedPod: unscheduled(withPVC(defaultPVCName, pod())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
{
name: "deleted running Pod -> finalizer is removed",
initialObjects: []runtime.Object{
deleted(withProtectionFinalizer(pvc())),
},
deletedPod: withStatus(v1.PodRunning, withPVC(defaultPVCName, pod())),
expectedActions: []clienttesting.Action{
clienttesting.NewUpdateAction(pvcVer, defaultNS, deleted(pvc())),
},
},
}
for _, test := range tests {
// Create client with initial data
objs := test.initialObjects
if test.updatedPVC != nil {
objs = append(objs, test.updatedPVC)
}
if test.updatedPod != nil {
objs = append(objs, test.updatedPod)
}
client := fake.NewSimpleClientset(objs...)
// Create informers
informers := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
pvcInformer := informers.Core().V1().PersistentVolumeClaims()
podInformer := informers.Core().V1().Pods()
// Populate the informers with initial objects so the controller can
// Get() and List() it.
for _, obj := range objs {
switch obj.(type) {
case *v1.PersistentVolumeClaim:
pvcInformer.Informer().GetStore().Add(obj)
case *v1.Pod:
podInformer.Informer().GetStore().Add(obj)
default:
t.Fatalf("Unknown initalObject type: %+v", obj)
}
}
// Add reactor to inject test errors.
for _, reactor := range test.reactors {
client.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorfn)
}
// Create the controller
ctrl := NewPVCProtectionController(pvcInformer, podInformer, client)
// Start the test by simulating an event
if test.updatedPVC != nil {
ctrl.pvcAddedUpdated(test.updatedPVC)
}
if test.updatedPod != nil {
ctrl.podAddedDeletedUpdated(test.updatedPod, false)
}
if test.deletedPod != nil {
ctrl.podAddedDeletedUpdated(test.deletedPod, true)
}
// Process the controller queue until we get expected results
timeout := time.Now().Add(10 * time.Second)
lastReportedActionCount := 0
for {
if time.Now().After(timeout) {
t.Errorf("Test %q: timed out", test.name)
break
}
if ctrl.queue.Len() > 0 {
glog.V(5).Infof("Test %q: %d events queue, processing one", test.name, ctrl.queue.Len())
ctrl.processNextWorkItem()
}
if ctrl.queue.Len() > 0 {
// There is still some work in the queue, process it now
continue
}
currentActionCount := len(client.Actions())
if currentActionCount < len(test.expectedActions) {
// Do not log evey wait, only when the action count changes.
if lastReportedActionCount < currentActionCount {
glog.V(5).Infof("Test %q: got %d actions out of %d, waiting for the rest", test.name, currentActionCount, len(test.expectedActions))
lastReportedActionCount = currentActionCount
}
// The test expected more to happen, wait for the actions.
// Most probably it's exponential backoff
time.Sleep(10 * time.Millisecond)
continue
}
break
}
actions := client.Actions()
for i, action := range actions {
if len(test.expectedActions) < i+1 {
t.Errorf("Test %q: %d unexpected actions: %+v", test.name, len(actions)-len(test.expectedActions), spew.Sdump(actions[i:]))
break
}
expectedAction := test.expectedActions[i]
if !reflect.DeepEqual(expectedAction, action) {
t.Errorf("Test %q: action %d\nExpected:\n%s\ngot:\n%s", test.name, i, spew.Sdump(expectedAction), spew.Sdump(action))
}
}
if len(test.expectedActions) > len(actions) {
t.Errorf("Test %q: %d additional expected actions", test.name, len(test.expectedActions)-len(actions))
for _, a := range test.expectedActions[len(actions):] {
t.Logf(" %+v", a)
}
}
}
}

7
pkg/features/kube_features.go
View File

@@ -200,6 +200,12 @@ const (
//
// Enable Block volume support in containers.
BlockVolume utilfeature.Feature = "BlockVolume"
// owner: @pospispa
//
// alpha: v1.9
// Postpone deletion of a persistent volume claim in case it is used by a pod
PVCProtection utilfeature.Feature = "PVCProtection"
)
func init() {
@@ -237,6 +243,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
CSIPersistentVolume: {Default: false, PreRelease: utilfeature.Alpha},
CustomPodDNS: {Default: false, PreRelease: utilfeature.Alpha},
BlockVolume: {Default: false, PreRelease: utilfeature.Alpha},
PVCProtection: {Default: false, PreRelease: utilfeature.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

33
pkg/kubelet/stats/cadvisor_stats_provider.go
View File

@@ -76,20 +76,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
if err != nil {
return nil, fmt.Errorf("failed to get imageFs info: %v", err)
}
infos, err := p.cadvisor.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2, // 2 samples are needed to compute "instantaneous" CPU
Recursive: true,
})
infos, err := getCadvisorContainerInfo(p.cadvisor)
if err != nil {
if _, ok := infos["/"]; ok {
// If the failure is partial, log it and return a best-effort
// response.
glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err)
} else {
return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
}
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
@@ -352,3 +341,21 @@ func hasMemoryAndCPUInstUsage(info *cadvisorapiv2.ContainerInfo) bool {
}
return cstat.CpuInst.Usage.Total != 0 && cstat.Memory.RSS != 0
}
func getCadvisorContainerInfo(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) {
infos, err := ca.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2, // 2 samples are needed to compute "instantaneous" CPU
Recursive: true,
})
if err != nil {
if _, ok := infos["/"]; ok {
// If the failure is partial, log it and return a best-effort
// response.
glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err)
} else {
return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
}
}
return infos, nil
}

77
pkg/kubelet/stats/cri_stats_provider.go
View File

@@ -18,7 +18,9 @@ package stats
import (
"fmt"
"path"
"sort"
"strings"
"time"
"github.com/golang/glog"
@@ -112,6 +114,11 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
containerMap[c.Id] = c
}
caInfos, err := getCRICadvisorStats(p.cadvisor)
if err != nil {
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
}
for _, stats := range resp {
containerID := stats.Attributes.Id
container, found := containerMap[containerID]
@@ -132,10 +139,25 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
ps, found := sandboxIDToPodStats[podSandboxID]
if !found {
ps = buildPodStats(podSandbox)
// Fill stats from cadvisor is available for full set of required pod stats
caPodSandbox, found := caInfos[podSandboxID]
if !found {
glog.V(4).Info("Unable to find cadvisor stats for sandbox %q", podSandboxID)
} else {
p.addCadvisorPodStats(ps, &caPodSandbox)
}
sandboxIDToPodStats[podSandboxID] = ps
}
containerStats := p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)
ps.Containers = append(ps.Containers, *containerStats)
cs := p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)
// If cadvisor stats is available for the container, use it to populate
// container stats
caStats, caFound := caInfos[containerID]
if !caFound {
glog.V(4).Info("Unable to find cadvisor stats for %q", containerID)
} else {
p.addCadvisorContainerStats(cs, &caStats)
}
ps.Containers = append(ps.Containers, *cs)
}
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
@@ -201,7 +223,7 @@ func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *c
return &fsInfo
}
// buildPodRef returns a PodStats that identifies the Pod managing cinfo
// buildPodStats returns a PodStats that identifies the Pod managing cinfo
func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
return &statsapi.PodStats{
PodRef: statsapi.PodReference{
@@ -211,7 +233,6 @@ func buildPodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
},
// The StartTime in the summary API is the pod creation time.
StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
// Network stats are not supported by CRI.
}
}
@@ -226,6 +247,13 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo
return s
}
func (p *criStatsProvider) addCadvisorPodStats(
ps *statsapi.PodStats,
caPodSandbox *cadvisorapiv2.ContainerInfo,
) {
ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, caPodSandbox)
}
func (p *criStatsProvider) makeContainerStats(
stats *runtimeapi.ContainerStats,
container *runtimeapi.Container,
@@ -336,3 +364,44 @@ func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi
}
return result
}
func (p *criStatsProvider) addCadvisorContainerStats(
cs *statsapi.ContainerStats,
caPodStats *cadvisorapiv2.ContainerInfo,
) {
if caPodStats.Spec.HasCustomMetrics {
cs.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(caPodStats)
}
cpu, memory := cadvisorInfoToCPUandMemoryStats(caPodStats)
if cpu != nil {
cs.CPU = cpu
}
if memory != nil {
cs.Memory = memory
}
}
func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) {
stats := make(map[string]cadvisorapiv2.ContainerInfo)
infos, err := getCadvisorContainerInfo(ca)
if err != nil {
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
infos = removeTerminatedContainerInfo(infos)
for key, info := range infos {
// On systemd using devicemapper each mount into the container has an
// associated cgroup. We ignore them to ensure we do not get duplicate
// entries in our summary. For details on .mount units:
// http://man7.org/linux/man-pages/man5/systemd.mount.5.html
if strings.HasSuffix(key, ".mount") {
continue
}
// Build the Pod key if this container is managed by a Pod
if !isPodManagedContainer(&info) {
continue
}
stats[path.Base(key)] = info
}
return stats, nil
}

108
pkg/kubelet/stats/cri_stats_provider_test.go
View File

@@ -22,20 +22,47 @@ import (
"time"
cadvisorfs "github.com/google/cadvisor/fs"
"github.com/stretchr/testify/assert"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/leaky"
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
)
func TestCRIListPodStats(t *testing.T) {
const (
seedRoot = 0
seedRuntime = 100
seedKubelet = 200
seedMisc = 300
seedSandbox0 = 1000
seedContainer0 = 2000
seedSandbox1 = 3000
seedContainer1 = 4000
seedContainer2 = 5000
seedSandbox2 = 6000
seedContainer3 = 7000
)
const (
pName0 = "pod0"
pName1 = "pod1"
pName2 = "pod2"
)
const (
cName0 = "container0-name"
cName1 = "container1-name"
cName2 = "container2-name"
cName3 = "container3-name"
)
var (
imageFsStorageUUID = "imagefs-storage-uuid"
unknownStorageUUID = "unknown-storage-uuid"
@@ -43,19 +70,19 @@ func TestCRIListPodStats(t *testing.T) {
rootFsInfo = getTestFsInfo(1000)
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
container0 = makeFakeContainer(sandbox0, "container0-name", 0, false)
container0 = makeFakeContainer(sandbox0, cName0, 0, false)
containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID)
container1 = makeFakeContainer(sandbox0, "container1-name", 0, false)
container1 = makeFakeContainer(sandbox0, cName1, 0, false)
containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID)
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
container2 = makeFakeContainer(sandbox1, "container2-name", 0, false)
container2 = makeFakeContainer(sandbox1, cName2, 0, false)
containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID)
sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
container3 = makeFakeContainer(sandbox2, "container3-name", 0, true)
container3 = makeFakeContainer(sandbox2, cName3, 0, true)
containerStats3 = makeFakeContainerStats(container3, imageFsStorageUUID)
container4 = makeFakeContainer(sandbox2, "container3-name", 1, false)
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
containerStats4 = makeFakeContainerStats(container4, imageFsStorageUUID)
)
@@ -68,7 +95,27 @@ func TestCRIListPodStats(t *testing.T) {
fakeImageService = critest.NewFakeImageService()
)
infos := map[string]cadvisorapiv2.ContainerInfo{
"/": getTestContainerInfo(seedRoot, "", "", ""),
"/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
"/system": getTestContainerInfo(seedMisc, "", "", ""),
sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
}
options := cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2,
Recursive: true,
}
mockCadvisor.
On("ContainerInfoV2", "/", options).Return(infos, nil).
On("RootFsInfo").Return(rootFsInfo, nil).
On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil).
On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
@@ -117,16 +164,18 @@ func TestCRIListPodStats(t *testing.T) {
for _, s := range p0.Containers {
containerStatsMap[s.Name] = s
}
c0 := containerStatsMap["container0-name"]
c0 := containerStatsMap[cName0]
assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c0, containerStats0)
checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
checkCRILogsStats(assert, c0, &rootFsInfo)
c1 := containerStatsMap["container1-name"]
c1 := containerStatsMap[cName1]
assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c1, containerStats1)
checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c1, containerStats1, nil)
checkCRILogsStats(assert, c1, &rootFsInfo)
checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
@@ -134,11 +183,12 @@ func TestCRIListPodStats(t *testing.T) {
checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
c2 := p1.Containers[0]
assert.Equal("container2-name", c2.Name)
assert.Equal(cName2, c2.Name)
assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c2, containerStats2)
checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
checkCRILogsStats(assert, c2, &rootFsInfo)
checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
@@ -147,12 +197,13 @@ func TestCRIListPodStats(t *testing.T) {
checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
c3 := p2.Containers[0]
assert.Equal("container3-name", c3.Name)
assert.Equal(cName3, c3.Name)
assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
checkCRICPUAndMemoryStats(assert, c3, containerStats4)
checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
checkCRILogsStats(assert, c3, &rootFsInfo)
checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
mockCadvisor.AssertExpectations(t)
}
@@ -306,18 +357,16 @@ func makeFakeVolumeStats(volumeNames []string) []statsapi.VolumeStats {
return volumes
}
func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
assert.Zero(*actual.CPU.UsageNanoCores)
func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *cadvisorapiv2.ContainerStats) {
assert.Equal(cs.Timestamp.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(cs.Cpu.Usage.Total, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(cs.CpuInst.Usage.Total, *actual.CPU.UsageNanoCores)
assert.Equal(cs.Memory.Timestamp, actual.Memory.Time.UnixNano())
assert.Nil(actual.Memory.AvailableBytes)
assert.Nil(actual.Memory.UsageBytes)
assert.Equal(cs.Memory.WorkingSetBytes.Value, *actual.Memory.WorkingSetBytes)
assert.Zero(*actual.Memory.RSSBytes)
assert.Nil(actual.Memory.PageFaults)
assert.Nil(actual.Memory.MajorPageFaults)
assert.Equal(cs.Memory.Usage, *actual.Memory.UsageBytes)
assert.Equal(cs.Memory.WorkingSet, *actual.Memory.WorkingSetBytes)
assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
}
func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
@@ -361,3 +410,10 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt
assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
}
func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
assert.Equal(expected.Interfaces[0].RxBytes, *actual.RxBytes)
assert.Equal(expected.Interfaces[0].RxErrors, *actual.RxErrors)
assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
}

47
pkg/kubelet/stats/helper.go
View File

@@ -86,29 +86,12 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
if rootFs != nil {
// The container logs live on the node rootfs device
result.Logs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &rootFs.Available,
CapacityBytes: &rootFs.Capacity,
InodesFree: rootFs.InodesFree,
Inodes: rootFs.Inodes,
}
if rootFs.Inodes != nil && rootFs.InodesFree != nil {
logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
result.Logs.InodesUsed = &logsInodesUsed
}
result.Logs = buildLogsStats(cstat, rootFs)
}
if imageFs != nil {
// The container rootFs lives on the imageFs devices (which may not be the node root fs)
result.Rootfs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &imageFs.Available,
CapacityBytes: &imageFs.Capacity,
InodesFree: imageFs.InodesFree,
Inodes: imageFs.Inodes,
}
result.Rootfs = buildRootfsStats(cstat, imageFs)
}
cfs := cstat.Filesystem
@@ -274,3 +257,29 @@ func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadviso
}
return stats, nil
}
func buildLogsStats(cstat *cadvisorapiv2.ContainerStats, rootFs *cadvisorapiv2.FsInfo) *statsapi.FsStats {
fsStats := &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &rootFs.Available,
CapacityBytes: &rootFs.Capacity,
InodesFree: rootFs.InodesFree,
Inodes: rootFs.Inodes,
}
if rootFs.Inodes != nil && rootFs.InodesFree != nil {
logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
fsStats.InodesUsed = &logsInodesUsed
}
return fsStats
}
func buildRootfsStats(cstat *cadvisorapiv2.ContainerStats, imageFs *cadvisorapiv2.FsInfo) *statsapi.FsStats {
return &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &imageFs.Available,
CapacityBytes: &imageFs.Capacity,
InodesFree: imageFs.InodesFree,
Inodes: imageFs.Inodes,
}
}

1
pkg/kubelet/volumemanager/populator/BUILD
View File

@@ -19,6 +19,7 @@ go_library(
"//pkg/kubelet/util/format:go_default_library",
"//pkg/kubelet/volumemanager/cache:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/types:go_default_library",
"//pkg/volume/util/volumehelper:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",

21
pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go
View File

@@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
"k8s.io/kubernetes/pkg/kubelet/volumemanager/cache"
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
@@ -419,7 +420,8 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec(
}
// getPVCExtractPV fetches the PVC object with the given namespace and name from
// the API server extracts the name of the PV it is pointing to and returns it.
// the API server, checks whether PVC is being deleted, extracts the name of the PV
// it is pointing to and returns it.
// An error is returned if the PVC object's phase is not "Bound".
func (dswp *desiredStateOfWorldPopulator) getPVCExtractPV(
namespace string, claimName string) (string, types.UID, error) {
@@ -433,6 +435,23 @@ func (dswp *desiredStateOfWorldPopulator) getPVCExtractPV(
err)
}
if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
// Pods that uses a PVC that is being deleted must not be started.
//
// In case an old kubelet is running without this check or some kubelets
// have this feature disabled, the worst that can happen is that such
// pod is scheduled. This was the default behavior in 1.8 and earlier
// and users should not be that surprised.
// It should happen only in very rare case when scheduler schedules
// a pod and user deletes a PVC that's used by it at the same time.
if volumeutil.IsPVCBeingDeleted(pvc) {
return "", "", fmt.Errorf(
"can't start pod because PVC %s/%s is being deleted",
namespace,
claimName)
}
}
if pvc.Status.Phase != v1.ClaimBound || pvc.Spec.VolumeName == "" {
return "", "", fmt.Errorf(

17
pkg/printers/internalversion/describe.go
View File

@@ -3386,6 +3386,9 @@ func describePodSecurityPolicy(psp *extensions.PodSecurityPolicy) (string, error
w.Write(LEVEL_1, "Allowed Capabilities:\t%s\n", capsToString(psp.Spec.AllowedCapabilities))
w.Write(LEVEL_1, "Allowed Volume Types:\t%s\n", fsTypeToString(psp.Spec.Volumes))
if len(psp.Spec.AllowedFlexVolumes) > 0 {
w.Write(LEVEL_1, "Allowed FlexVolume Types:\t%s\n", flexVolumesToString(psp.Spec.AllowedFlexVolumes))
}
w.Write(LEVEL_1, "Allow Host Network:\t%t\n", psp.Spec.HostNetwork)
w.Write(LEVEL_1, "Allow Host Ports:\t%s\n", hostPortRangeToString(psp.Spec.HostPorts))
w.Write(LEVEL_1, "Allow Host PID:\t%t\n", psp.Spec.HostPID)
@@ -3419,10 +3422,14 @@ func describePodSecurityPolicy(psp *extensions.PodSecurityPolicy) (string, error
}
func stringOrNone(s string) string {
return stringOrDefaultValue(s, "<none>")
}
func stringOrDefaultValue(s, defaultValue string) string {
if len(s) > 0 {
return s
}
return "<none>"
return defaultValue
}
func fsTypeToString(volumes []extensions.FSType) string {
@@ -3433,6 +3440,14 @@ func fsTypeToString(volumes []extensions.FSType) string {
return stringOrNone(strings.Join(strVolumes, ","))
}
func flexVolumesToString(flexVolumes []extensions.AllowedFlexVolume) string {
volumes := []string{}
for _, flexVolume := range flexVolumes {
volumes = append(volumes, "driver="+flexVolume.Driver)
}
return stringOrDefaultValue(strings.Join(volumes, ","), "<all>")
}
func hostPortRangeToString(ranges []extensions.HostPortRange) string {
formattedString := ""
if ranges != nil {

1
pkg/registry/rbac/rest/storage_rbac.go
View File

@@ -343,6 +343,7 @@ func primeAggregatedClusterRoles(clusterRolesToAggregate map[string]string, clus
}
glog.V(1).Infof("migrating %v to %v", existingRole.Name, newName)
existingRole.Name = newName
existingRole.ResourceVersion = "" // clear this so the object can be created.
if _, err := clusterRoleClient.ClusterRoles().Create(existingRole); err != nil && !apierrors.IsAlreadyExists(err) {
return err
}

17
pkg/security/podsecuritypolicy/provider.go
View File

@@ -233,9 +233,24 @@ func (s *simpleProvider) ValidatePodSecurityContext(pod *api.Pod, fldPath *field
fmt.Sprintf("is not allowed to be used")))
}
}
if fsType == extensions.FlexVolume && len(s.psp.Spec.AllowedFlexVolumes) > 0 {
found := false
driver := v.FlexVolume.Driver
for _, allowedFlexVolume := range s.psp.Spec.AllowedFlexVolumes {
if driver == allowedFlexVolume.Driver {
found = true
break
}
}
if !found {
allErrs = append(allErrs,
field.Invalid(fldPath.Child("volumes").Index(i).Child("driver"), driver,
"Flexvolume driver is not allowed to be used"))
}
}
}
}
return allErrs
}

72
pkg/security/podsecuritypolicy/provider_test.go
View File

@@ -256,6 +256,18 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
failSeccompProfilePod := defaultPod()
failSeccompProfilePod.Annotations = map[string]string{api.SeccompPodAnnotationKey: "foo"}
podWithInvalidFlexVolumeDriver := defaultPod()
podWithInvalidFlexVolumeDriver.Spec.Volumes = []api.Volume{
{
Name: "flex-volume",
VolumeSource: api.VolumeSource{
FlexVolume: &api.FlexVolumeSource{
Driver: "example/unknown",
},
},
},
}
errorCases := map[string]struct {
pod *api.Pod
psp *extensions.PodSecurityPolicy
@@ -341,6 +353,16 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
psp: defaultPSP(),
expectedError: "Forbidden: seccomp may not be set",
},
"fail pod with disallowed flexVolume when flex volumes are allowed": {
pod: podWithInvalidFlexVolumeDriver,
psp: allowFlexVolumesPSP(false, false),
expectedError: "Flexvolume driver is not allowed to be used",
},
"fail pod with disallowed flexVolume when all volumes are allowed": {
pod: podWithInvalidFlexVolumeDriver,
psp: allowFlexVolumesPSP(false, true),
expectedError: "Flexvolume driver is not allowed to be used",
},
}
for k, v := range errorCases {
provider, err := NewSimpleProvider(v.psp, "namespace", NewSimpleStrategyFactory())
@@ -358,6 +380,28 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
}
}
func allowFlexVolumesPSP(allowAllFlexVolumes, allowAllVolumes bool) *extensions.PodSecurityPolicy {
psp := defaultPSP()
allowedVolumes := []extensions.AllowedFlexVolume{
{Driver: "example/foo"},
{Driver: "example/bar"},
}
if allowAllFlexVolumes {
allowedVolumes = []extensions.AllowedFlexVolume{}
}
allowedVolumeType := extensions.FlexVolume
if allowAllVolumes {
allowedVolumeType = extensions.All
}
psp.Spec.AllowedFlexVolumes = allowedVolumes
psp.Spec.Volumes = []extensions.FSType{allowedVolumeType}
return psp
}
func TestValidateContainerSecurityContextFailures(t *testing.T) {
// fail user strat
failUserPSP := defaultPSP()
@@ -597,6 +641,18 @@ func TestValidatePodSecurityContextSuccess(t *testing.T) {
api.SeccompPodAnnotationKey: "foo",
}
flexVolumePod := defaultPod()
flexVolumePod.Spec.Volumes = []api.Volume{
{
Name: "flex-volume",
VolumeSource: api.VolumeSource{
FlexVolume: &api.FlexVolumeSource{
Driver: "example/bar",
},
},
},
}
successCases := map[string]struct {
pod *api.Pod
psp *extensions.PodSecurityPolicy
@@ -653,6 +709,22 @@ func TestValidatePodSecurityContextSuccess(t *testing.T) {
pod: seccompPod,
psp: seccompPSP,
},
"flex volume driver in a whitelist (all volumes are allowed)": {
pod: flexVolumePod,
psp: allowFlexVolumesPSP(false, true),
},
"flex volume driver with empty whitelist (all volumes are allowed)": {
pod: flexVolumePod,
psp: allowFlexVolumesPSP(true, true),
},
"flex volume driver in a whitelist (only flex volumes are allowed)": {
pod: flexVolumePod,
psp: allowFlexVolumesPSP(false, false),
},
"flex volume driver with empty whitelist (only flex volumes volumes are allowed)": {
pod: flexVolumePod,
psp: allowFlexVolumesPSP(true, false),
},
}
for k, v := range successCases {

1
pkg/volume/aws_ebs/BUILD
View File

@@ -46,6 +46,7 @@ go_test(
"//pkg/volume/testing:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",

8
pkg/volume/aws_ebs/attacher_test.go
View File

@@ -26,6 +26,7 @@ import (
volumetest "k8s.io/kubernetes/pkg/volume/testing"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
)
@@ -340,3 +341,10 @@ func (testcase *testcase) GetVolumeLabels(volumeName aws.KubernetesVolumeID) (ma
func (testcase *testcase) GetDiskPath(volumeName aws.KubernetesVolumeID) (string, error) {
return "", errors.New("Not implemented")
}
func (testcase *testcase) ResizeDisk(
volumeName aws.KubernetesVolumeID,
oldSize resource.Quantity,
newSize resource.Quantity) (resource.Quantity, error) {
return oldSize, errors.New("Not implemented")
}

27
pkg/volume/aws_ebs/aws_ebs.go
View File

@@ -241,6 +241,33 @@ func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath
return volume.NewSpecFromVolume(awsVolume), nil
}
func (plugin *awsElasticBlockStorePlugin) RequiresFSResize() bool {
return true
}
func (plugin *awsElasticBlockStorePlugin) ExpandVolumeDevice(
spec *volume.Spec,
newSize resource.Quantity,
oldSize resource.Quantity) (resource.Quantity, error) {
var awsVolume aws.Volumes
awsVolume, err := getCloudProvider(plugin.host.GetCloudProvider())
if err != nil {
return oldSize, err
}
// we don't expect to receive this call for non PVs
rawVolumeName := spec.PersistentVolume.Spec.AWSElasticBlockStore.VolumeID
volumeID := aws.KubernetesVolumeID(rawVolumeName)
if volumeID == "" {
return oldSize, fmt.Errorf("EBS.ExpandVolumeDevice Invalid volume id for %s", spec.Name())
}
return awsVolume.ResizeDisk(volumeID, oldSize, newSize)
}
var _ volume.ExpandableVolumePlugin = &awsElasticBlockStorePlugin{}
// Abstract interface to PD operations.
type ebsManager interface {
CreateVolume(provisioner *awsElasticBlockStoreProvisioner) (volumeID aws.KubernetesVolumeID, volumeSizeGB int, labels map[string]string, fstype string, err error)

6
pkg/volume/downwardapi/downwardapi_test.go
View File

@@ -63,6 +63,12 @@ func TestCanSupport(t *testing.T) {
if plugin.GetPluginName() != downwardAPIPluginName {
t.Errorf("Wrong name: %s", plugin.GetPluginName())
}
if !plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{DownwardAPI: &v1.DownwardAPIVolumeSource{}}}}) {
t.Errorf("Expected true")
}
if plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) {
t.Errorf("Expected false")
}
}
func TestDownwardAPI(t *testing.T) {

3
pkg/volume/util/BUILD
View File

@@ -14,6 +14,7 @@ go_library(
"device_util_unsupported.go",
"doc.go",
"error.go",
"finalizer.go",
"fs_unsupported.go",
"io_util.go",
"metrics.go",
@@ -61,6 +62,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
"finalizer_test.go",
"util_test.go",
] + select({
"@io_bazel_rules_go//go/platform:linux_amd64": [
@@ -74,6 +76,7 @@ go_test(
deps = [
"//pkg/apis/core/install:go_default_library",
"//pkg/apis/core/v1/helper:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

68
pkg/volume/util/finalizer.go Normal file
View File

@@ -0,0 +1,68 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"k8s.io/api/core/v1"
)
const (
// Name of finalizer on PVCs that have a running pod.
PVCProtectionFinalizer = "kubernetes.io/pvc-protection"
)
// IsPVCBeingDeleted returns:
// true: in case PVC is being deleted, i.e. ObjectMeta.DeletionTimestamp is set
// false: in case PVC is not being deleted, i.e. ObjectMeta.DeletionTimestamp is nil
func IsPVCBeingDeleted(pvc *v1.PersistentVolumeClaim) bool {
return pvc.ObjectMeta.DeletionTimestamp != nil
}
// IsProtectionFinalizerPresent returns true in case PVCProtectionFinalizer is
// present among the pvc.Finalizers
func IsProtectionFinalizerPresent(pvc *v1.PersistentVolumeClaim) bool {
for _, finalizer := range pvc.Finalizers {
if finalizer == PVCProtectionFinalizer {
return true
}
}
return false
}
// RemoveProtectionFinalizer returns pvc without PVCProtectionFinalizer in case
// it's present in pvc.Finalizers. It expects that pvc is writable (i.e. is not
// informer's cached copy.)
func RemoveProtectionFinalizer(pvc *v1.PersistentVolumeClaim) {
newFinalizers := make([]string, 0)
for _, finalizer := range pvc.Finalizers {
if finalizer != PVCProtectionFinalizer {
newFinalizers = append(newFinalizers, finalizer)
}
}
if len(newFinalizers) == 0 {
// Sanitize for unit tests so we don't need to distinguish empty array
// and nil.
newFinalizers = nil
}
pvc.Finalizers = newFinalizers
}
// AddProtectionFinalizer adds PVCProtectionFinalizer to pvc. It expects that
// pvc is writable (i.e. is not informer's cached copy.)
func AddProtectionFinalizer(pvc *v1.PersistentVolumeClaim) {
pvc.Finalizers = append(pvc.Finalizers, PVCProtectionFinalizer)
}

231
pkg/volume/util/finalizer_test.go Normal file
View File

@@ -0,0 +1,231 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
arbitraryTime = metav1.Date(2017, 11, 1, 14, 28, 47, 0, time.FixedZone("CET", 0))
)
func TestIsPVCBeingDeleted(t *testing.T) {
tests := []struct {
pvc *v1.PersistentVolumeClaim
want bool
}{
{
pvc: &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
DeletionTimestamp: nil,
},
},
want: false,
},
{
pvc: &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
DeletionTimestamp: &arbitraryTime,
},
},
want: true,
},
}
for _, tt := range tests {
if got := IsPVCBeingDeleted(tt.pvc); got != tt.want {
t.Errorf("IsPVCBeingDeleted(%v) = %v WANT %v", tt.pvc, got, tt.want)
}
}
}
func TestAddProtectionFinalizer(t *testing.T) {
tests := []struct {
name string
pvc *v1.PersistentVolumeClaim
want *v1.PersistentVolumeClaim
}{
{
"PVC without finalizer",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
},
},
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{PVCProtectionFinalizer},
},
},
},
{
"PVC with some finalizers",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
},
},
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
},
},
},
}
for _, test := range tests {
got := test.pvc.DeepCopy()
AddProtectionFinalizer(got)
if !reflect.DeepEqual(got, test.want) {
t.Errorf("Test %q: expected:\n%s\n\ngot:\n%s", test.name, spew.Sdump(test.want), spew.Sdump(got))
}
}
}
func TestRemoveProtectionFinalizer(t *testing.T) {
tests := []struct {
name string
pvc *v1.PersistentVolumeClaim
want *v1.PersistentVolumeClaim
}{
{
"PVC without finalizer",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
},
},
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
},
},
},
{
"PVC with finalizer",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{PVCProtectionFinalizer},
},
},
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
},
},
},
{
"PVC with many finalizers",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
},
},
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
},
},
},
}
for _, test := range tests {
got := test.pvc.DeepCopy()
RemoveProtectionFinalizer(got)
if !reflect.DeepEqual(got, test.want) {
t.Errorf("Test %q: expected:\n%s\n\ngot:\n%s", test.name, spew.Sdump(test.want), spew.Sdump(got))
}
}
}
func TestIsProtectionFinalizerPresent(t *testing.T) {
tests := []struct {
name string
pvc *v1.PersistentVolumeClaim
want bool
}{
{
"PVC without finalizer",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
},
},
false,
},
{
"PVC with many unrelated finalizers",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer},
},
},
false,
},
{
"PVC with many finalizers",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{"1", "2", "3", PVCProtectionFinalizer + "suffix", "prefix" + PVCProtectionFinalizer, PVCProtectionFinalizer},
},
},
true,
},
{
"PVC with finalizer",
&v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "pvc",
Namespace: "ns",
Finalizers: []string{PVCProtectionFinalizer},
},
},
true,
},
}
for _, test := range tests {
got := IsProtectionFinalizerPresent(test.pvc)
if got != test.want {
t.Errorf("Test %q: expected %v, got %v", test.name, test.want, got)
}
}
}

1
plugin/BUILD
View File

@@ -29,6 +29,7 @@ filegroup(
"//plugin/pkg/admission/noderestriction:all-srcs",
"//plugin/pkg/admission/persistentvolume/label:all-srcs",
"//plugin/pkg/admission/persistentvolume/resize:all-srcs",
"//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:all-srcs",
"//plugin/pkg/admission/podnodeselector:all-srcs",
"//plugin/pkg/admission/podpreset:all-srcs",
"//plugin/pkg/admission/podtolerationrestriction:all-srcs",

1
plugin/pkg/admission/persistentvolume/label/BUILD
View File

@@ -34,6 +34,7 @@ go_test(
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/cloudprovider/providers/aws:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",

8
plugin/pkg/admission/persistentvolume/label/admission_test.go
View File

@@ -21,6 +21,7 @@ import (
"fmt"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/admission"
@@ -67,6 +68,13 @@ func (c *mockVolumes) DisksAreAttached(nodeDisks map[types.NodeName][]aws.Kubern
return nil, fmt.Errorf("not implemented")
}
func (c *mockVolumes) ResizeDisk(
diskName aws.KubernetesVolumeID,
oldSize resource.Quantity,
newSize resource.Quantity) (resource.Quantity, error) {
return oldSize, nil
}
func mockVolumeFailure(err error) *mockVolumes {
return &mockVolumes{volumeLabelsError: err}
}

6
plugin/pkg/admission/persistentvolume/resize/admission.go
View File

@@ -156,6 +156,10 @@ func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVol
if pv.Spec.GCEPersistentDisk != nil {
return true
}
return false
if pv.Spec.AWSElasticBlockStore != nil {
return true
}
return false
}

51
plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD Normal file
View File

@@ -0,0 +1,51 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["admission.go"],
importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection",
visibility = ["//visibility:public"],
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/client/informers/informers_generated/internalversion:go_default_library",
"//pkg/client/listers/core/internalversion:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubeapiserver/admission:go_default_library",
"//pkg/volume/util:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["admission_test.go"],
importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection",
library = ":go_default_library",
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/client/informers/informers_generated/internalversion:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/volume/util:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

111
plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go Normal file
View File

@@ -0,0 +1,111 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pvcprotection
import (
"fmt"
"io"
"github.com/golang/glog"
admission "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/util/feature"
api "k8s.io/kubernetes/pkg/apis/core"
informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion"
"k8s.io/kubernetes/pkg/features"
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
)
const (
// PluginName is the name of this admission controller plugin
PluginName = "PVCProtection"
)
// Register registers a plugin
func Register(plugins *admission.Plugins) {
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
plugin := newPlugin()
return plugin, nil
})
}
// pvcProtectionPlugin holds state for and implements the admission plugin.
type pvcProtectionPlugin struct {
*admission.Handler
lister corelisters.PersistentVolumeClaimLister
}
var _ admission.Interface = &pvcProtectionPlugin{}
var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&pvcProtectionPlugin{})
// newPlugin creates a new admission plugin.
func newPlugin() *pvcProtectionPlugin {
return &pvcProtectionPlugin{
Handler: admission.NewHandler(admission.Create),
}
}
func (c *pvcProtectionPlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) {
informer := f.Core().InternalVersion().PersistentVolumeClaims()
c.lister = informer.Lister()
c.SetReadyFunc(informer.Informer().HasSynced)
}
// ValidateInitialization ensures lister is set.
func (c *pvcProtectionPlugin) ValidateInitialization() error {
if c.lister == nil {
return fmt.Errorf("missing lister")
}
return nil
}
// Admit sets finalizer on all PVCs. The finalizer is removed by
// PVCProtectionController when it's not referenced by any pod.
//
// This prevents users from deleting a PVC that's used by a running pod.
func (c *pvcProtectionPlugin) Admit(a admission.Attributes) error {
if !feature.DefaultFeatureGate.Enabled(features.PVCProtection) {
return nil
}
if a.GetResource().GroupResource() != api.Resource("persistentvolumeclaims") {
return nil
}
if len(a.GetSubresource()) != 0 {
return nil
}
pvc, ok := a.GetObject().(*api.PersistentVolumeClaim)
// if we can't convert then we don't handle this object so just return
if !ok {
return nil
}
for _, f := range pvc.Finalizers {
if f == volumeutil.PVCProtectionFinalizer {
// Finalizer is already present, nothing to do
return nil
}
}
glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name)
pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer)
return nil
}

106
plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go Normal file
View File

@@ -0,0 +1,106 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pvcprotection
import (
"fmt"
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/util/feature"
api "k8s.io/kubernetes/pkg/apis/core"
informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
"k8s.io/kubernetes/pkg/controller"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
)
func TestAdmit(t *testing.T) {
claim := &api.PersistentVolumeClaim{
TypeMeta: metav1.TypeMeta{
Kind: "PersistentVolumeClaim",
},
ObjectMeta: metav1.ObjectMeta{
Name: "claim",
Namespace: "ns",
},
}
claimWithFinalizer := claim.DeepCopy()
claimWithFinalizer.Finalizers = []string{volumeutil.PVCProtectionFinalizer}
tests := []struct {
name string
object runtime.Object
expectedObject runtime.Object
featureEnabled bool
}{
{
"create -> add finalizer",
claim,
claimWithFinalizer,
true,
},
{
"finalizer already exists -> no new finalizer",
claimWithFinalizer,
claimWithFinalizer,
true,
},
{
"disabled feature -> no finalizer",
claim,
claim,
false,
},
}
ctrl := newPlugin()
informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())
ctrl.SetInternalKubeInformerFactory(informerFactory)
for _, test := range tests {
feature.DefaultFeatureGate.Set(fmt.Sprintf("PVCProtection=%v", test.featureEnabled))
obj := test.object.DeepCopyObject()
attrs := admission.NewAttributesRecord(
obj, // new object
obj.DeepCopyObject(), // old object, copy to be sure it's not modified
api.Kind("PersistentVolumeClaim").WithVersion("version"),
claim.Namespace,
claim.Name,
api.Resource("persistentvolumeclaims").WithVersion("version"),
"", // subresource
admission.Create,
nil, // userInfo
)
err := ctrl.Admit(attrs)
if err != nil {
t.Errorf("Test %q: got unexpected error: %v", test.name, err)
}
if !reflect.DeepEqual(test.expectedObject, obj) {
t.Errorf("Test %q: Expected object:\n%s\ngot:\n%s", test.name, spew.Sdump(test.expectedObject), spew.Sdump(obj))
}
}
// Disable the feature for rest of the tests.
// TODO: remove after alpha
feature.DefaultFeatureGate.Set("PVCProtection=false")
}

10
plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go
View File

@@ -315,6 +315,16 @@ func buildControllerRoles() ([]rbac.ClusterRole, []rbac.ClusterRoleBinding) {
eventsRule(),
},
})
if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
addControllerRole(&controllerRoles, &controllerRoleBindings, rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "pvc-protection-controller"},
Rules: []rbac.PolicyRule{
rbac.NewRule("get", "list", "watch", "update").Groups(legacyGroup).Resources("persistentvolumeclaims").RuleOrDie(),
rbac.NewRule("list", "watch", "get").Groups(legacyGroup).Resources("pods").RuleOrDie(),
eventsRule(),
},
})
}
return controllerRoles, controllerRoleBindings
}

730
staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go generated
View File

@@ -25,6 +25,7 @@ limitations under the License.
k8s.io/kubernetes/vendor/k8s.io/api/extensions/v1beta1/generated.proto
It has these top-level messages:
AllowedFlexVolume
AllowedHostPath
CustomMetricCurrentStatus
CustomMetricCurrentStatusList
@@ -113,253 +114,258 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
func (m *AllowedFlexVolume) Reset() { *m = AllowedFlexVolume{} }
func (*AllowedFlexVolume) ProtoMessage() {}
func (*AllowedFlexVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} }
func (m *AllowedHostPath) Reset() { *m = AllowedHostPath{} }
func (*AllowedHostPath) ProtoMessage() {}
func (*AllowedHostPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} }
func (*AllowedHostPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{1} }
func (m *CustomMetricCurrentStatus) Reset() { *m = CustomMetricCurrentStatus{} }
func (*CustomMetricCurrentStatus) ProtoMessage() {}
func (*CustomMetricCurrentStatus) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{1}
return fileDescriptorGenerated, []int{2}
}
func (m *CustomMetricCurrentStatusList) Reset() { *m = CustomMetricCurrentStatusList{} }
func (*CustomMetricCurrentStatusList) ProtoMessage() {}
func (*CustomMetricCurrentStatusList) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{2}
return fileDescriptorGenerated, []int{3}
}
func (m *CustomMetricTarget) Reset() { *m = CustomMetricTarget{} }
func (*CustomMetricTarget) ProtoMessage() {}
func (*CustomMetricTarget) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} }
func (*CustomMetricTarget) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} }
func (m *CustomMetricTargetList) Reset() { *m = CustomMetricTargetList{} }
func (*CustomMetricTargetList) ProtoMessage() {}
func (*CustomMetricTargetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} }
func (*CustomMetricTargetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} }
func (m *DaemonSet) Reset() { *m = DaemonSet{} }
func (*DaemonSet) ProtoMessage() {}
func (*DaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} }
func (*DaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} }
func (m *DaemonSetCondition) Reset() { *m = DaemonSetCondition{} }
func (*DaemonSetCondition) ProtoMessage() {}
func (*DaemonSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} }
func (*DaemonSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} }
func (m *DaemonSetList) Reset() { *m = DaemonSetList{} }
func (*DaemonSetList) ProtoMessage() {}
func (*DaemonSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} }
func (*DaemonSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} }
func (m *DaemonSetSpec) Reset() { *m = DaemonSetSpec{} }
func (*DaemonSetSpec) ProtoMessage() {}
func (*DaemonSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} }
func (*DaemonSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{9} }
func (m *DaemonSetStatus) Reset() { *m = DaemonSetStatus{} }
func (*DaemonSetStatus) ProtoMessage() {}
func (*DaemonSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{9} }
func (*DaemonSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{10} }
func (m *DaemonSetUpdateStrategy) Reset() { *m = DaemonSetUpdateStrategy{} }
func (*DaemonSetUpdateStrategy) ProtoMessage() {}
func (*DaemonSetUpdateStrategy) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{10}
return fileDescriptorGenerated, []int{11}
}
func (m *Deployment) Reset() { *m = Deployment{} }
func (*Deployment) ProtoMessage() {}
func (*Deployment) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} }
func (*Deployment) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
func (m *DeploymentCondition) Reset() { *m = DeploymentCondition{} }
func (*DeploymentCondition) ProtoMessage() {}
func (*DeploymentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} }
func (*DeploymentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
func (m *DeploymentList) Reset() { *m = DeploymentList{} }
func (*DeploymentList) ProtoMessage() {}
func (*DeploymentList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} }
func (*DeploymentList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
func (m *DeploymentRollback) Reset() { *m = DeploymentRollback{} }
func (*DeploymentRollback) ProtoMessage() {}
func (*DeploymentRollback) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} }
func (*DeploymentRollback) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} }
func (m *DeploymentSpec) Reset() { *m = DeploymentSpec{} }
func (*DeploymentSpec) ProtoMessage() {}
func (*DeploymentSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} }
func (*DeploymentSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} }
func (m *DeploymentStatus) Reset() { *m = DeploymentStatus{} }
func (*DeploymentStatus) ProtoMessage() {}
func (*DeploymentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} }
func (*DeploymentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} }
func (m *DeploymentStrategy) Reset() { *m = DeploymentStrategy{} }
func (*DeploymentStrategy) ProtoMessage() {}
func (*DeploymentStrategy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} }
func (*DeploymentStrategy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} }
func (m *FSGroupStrategyOptions) Reset() { *m = FSGroupStrategyOptions{} }
func (*FSGroupStrategyOptions) ProtoMessage() {}
func (*FSGroupStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} }
func (*FSGroupStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} }
func (m *HTTPIngressPath) Reset() { *m = HTTPIngressPath{} }
func (*HTTPIngressPath) ProtoMessage() {}
func (*HTTPIngressPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} }
func (*HTTPIngressPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} }
func (m *HTTPIngressRuleValue) Reset() { *m = HTTPIngressRuleValue{} }
func (*HTTPIngressRuleValue) ProtoMessage() {}
func (*HTTPIngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} }
func (*HTTPIngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} }
func (m *HostPortRange) Reset() { *m = HostPortRange{} }
func (*HostPortRange) ProtoMessage() {}
func (*HostPortRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} }
func (*HostPortRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} }
func (m *IDRange) Reset() { *m = IDRange{} }
func (*IDRange) ProtoMessage() {}
func (*IDRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} }
func (*IDRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} }
func (m *IPBlock) Reset() { *m = IPBlock{} }
func (*IPBlock) ProtoMessage() {}
func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} }
func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} }
func (m *Ingress) Reset() { *m = Ingress{} }
func (*Ingress) ProtoMessage() {}
func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} }
func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} }
func (m *IngressBackend) Reset() { *m = IngressBackend{} }
func (*IngressBackend) ProtoMessage() {}
func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} }
func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} }
func (m *IngressList) Reset() { *m = IngressList{} }
func (*IngressList) ProtoMessage() {}
func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} }
func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} }
func (m *IngressRule) Reset() { *m = IngressRule{} }
func (*IngressRule) ProtoMessage() {}
func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} }
func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} }
func (m *IngressRuleValue) Reset() { *m = IngressRuleValue{} }
func (*IngressRuleValue) ProtoMessage() {}
func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} }
func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} }
func (m *IngressSpec) Reset() { *m = IngressSpec{} }
func (*IngressSpec) ProtoMessage() {}
func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} }
func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} }
func (m *IngressStatus) Reset() { *m = IngressStatus{} }
func (*IngressStatus) ProtoMessage() {}
func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} }
func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} }
func (m *IngressTLS) Reset() { *m = IngressTLS{} }
func (*IngressTLS) ProtoMessage() {}
func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} }
func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} }
func (m *NetworkPolicy) Reset() { *m = NetworkPolicy{} }
func (*NetworkPolicy) ProtoMessage() {}
func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} }
func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} }
func (m *NetworkPolicyEgressRule) Reset() { *m = NetworkPolicyEgressRule{} }
func (*NetworkPolicyEgressRule) ProtoMessage() {}
func (*NetworkPolicyEgressRule) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{33}
return fileDescriptorGenerated, []int{34}
}
func (m *NetworkPolicyIngressRule) Reset() { *m = NetworkPolicyIngressRule{} }
func (*NetworkPolicyIngressRule) ProtoMessage() {}
func (*NetworkPolicyIngressRule) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{34}
return fileDescriptorGenerated, []int{35}
}
func (m *NetworkPolicyList) Reset() { *m = NetworkPolicyList{} }
func (*NetworkPolicyList) ProtoMessage() {}
func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} }
func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} }
func (m *NetworkPolicyPeer) Reset() { *m = NetworkPolicyPeer{} }
func (*NetworkPolicyPeer) ProtoMessage() {}
func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} }
func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} }
func (m *NetworkPolicyPort) Reset() { *m = NetworkPolicyPort{} }
func (*NetworkPolicyPort) ProtoMessage() {}
func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} }
func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} }
func (m *NetworkPolicySpec) Reset() { *m = NetworkPolicySpec{} }
func (*NetworkPolicySpec) ProtoMessage() {}
func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} }
func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} }
func (m *PodSecurityPolicy) Reset() { *m = PodSecurityPolicy{} }
func (*PodSecurityPolicy) ProtoMessage() {}
func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} }
func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} }
func (m *PodSecurityPolicyList) Reset() { *m = PodSecurityPolicyList{} }
func (*PodSecurityPolicyList) ProtoMessage() {}
func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} }
func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} }
func (m *PodSecurityPolicySpec) Reset() { *m = PodSecurityPolicySpec{} }
func (*PodSecurityPolicySpec) ProtoMessage() {}
func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} }
func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} }
func (m *ReplicaSet) Reset() { *m = ReplicaSet{} }
func (*ReplicaSet) ProtoMessage() {}
func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} }
func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} }
func (m *ReplicaSetCondition) Reset() { *m = ReplicaSetCondition{} }
func (*ReplicaSetCondition) ProtoMessage() {}
func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} }
func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} }
func (m *ReplicaSetList) Reset() { *m = ReplicaSetList{} }
func (*ReplicaSetList) ProtoMessage() {}
func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} }
func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} }
func (m *ReplicaSetSpec) Reset() { *m = ReplicaSetSpec{} }
func (*ReplicaSetSpec) ProtoMessage() {}
func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} }
func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} }
func (m *ReplicaSetStatus) Reset() { *m = ReplicaSetStatus{} }
func (*ReplicaSetStatus) ProtoMessage() {}
func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} }
func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} }
func (m *ReplicationControllerDummy) Reset() { *m = ReplicationControllerDummy{} }
func (*ReplicationControllerDummy) ProtoMessage() {}
func (*ReplicationControllerDummy) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{47}
return fileDescriptorGenerated, []int{48}
}
func (m *RollbackConfig) Reset() { *m = RollbackConfig{} }
func (*RollbackConfig) ProtoMessage() {}
func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{48} }
func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} }
func (m *RollingUpdateDaemonSet) Reset() { *m = RollingUpdateDaemonSet{} }
func (*RollingUpdateDaemonSet) ProtoMessage() {}
func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} }
func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} }
func (m *RollingUpdateDeployment) Reset() { *m = RollingUpdateDeployment{} }
func (*RollingUpdateDeployment) ProtoMessage() {}
func (*RollingUpdateDeployment) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{50}
return fileDescriptorGenerated, []int{51}
}
func (m *RunAsUserStrategyOptions) Reset() { *m = RunAsUserStrategyOptions{} }
func (*RunAsUserStrategyOptions) ProtoMessage() {}
func (*RunAsUserStrategyOptions) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{51}
return fileDescriptorGenerated, []int{52}
}
func (m *SELinuxStrategyOptions) Reset() { *m = SELinuxStrategyOptions{} }
func (*SELinuxStrategyOptions) ProtoMessage() {}
func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} }
func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} }
func (m *Scale) Reset() { *m = Scale{} }
func (*Scale) ProtoMessage() {}
func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} }
func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} }
func (m *ScaleSpec) Reset() { *m = ScaleSpec{} }
func (*ScaleSpec) ProtoMessage() {}
func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} }
func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} }
func (m *ScaleStatus) Reset() { *m = ScaleStatus{} }
func (*ScaleStatus) ProtoMessage() {}
func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} }
func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} }
func (m *SupplementalGroupsStrategyOptions) Reset() { *m = SupplementalGroupsStrategyOptions{} }
func (*SupplementalGroupsStrategyOptions) ProtoMessage() {}
func (*SupplementalGroupsStrategyOptions) Descriptor() ([]byte, []int) {
return fileDescriptorGenerated, []int{56}
return fileDescriptorGenerated, []int{57}
}
func init() {
proto.RegisterType((*AllowedFlexVolume)(nil), "k8s.io.api.extensions.v1beta1.AllowedFlexVolume")
proto.RegisterType((*AllowedHostPath)(nil), "k8s.io.api.extensions.v1beta1.AllowedHostPath")
proto.RegisterType((*CustomMetricCurrentStatus)(nil), "k8s.io.api.extensions.v1beta1.CustomMetricCurrentStatus")
proto.RegisterType((*CustomMetricCurrentStatusList)(nil), "k8s.io.api.extensions.v1beta1.CustomMetricCurrentStatusList")
@@ -418,6 +424,28 @@ func init() {
proto.RegisterType((*ScaleStatus)(nil), "k8s.io.api.extensions.v1beta1.ScaleStatus")
proto.RegisterType((*SupplementalGroupsStrategyOptions)(nil), "k8s.io.api.extensions.v1beta1.SupplementalGroupsStrategyOptions")
}
func (m *AllowedFlexVolume) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *AllowedFlexVolume) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
dAtA[i] = 0xa
i++
i = encodeVarintGenerated(dAtA, i, uint64(len(m.Driver)))
i += copy(dAtA[i:], m.Driver)
return i, nil
}
func (m *AllowedHostPath) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@@ -2207,6 +2235,20 @@ func (m *PodSecurityPolicySpec) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if len(m.AllowedFlexVolumes) > 0 {
for _, msg := range m.AllowedFlexVolumes {
dAtA[i] = 0x92
i++
dAtA[i] = 0x1
i++
i = encodeVarintGenerated(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
return i, nil
}
@@ -2763,6 +2805,14 @@ func encodeVarintGenerated(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *AllowedFlexVolume) Size() (n int) {
var l int
_ = l
l = len(m.Driver)
n += 1 + l + sovGenerated(uint64(l))
return n
}
func (m *AllowedHostPath) Size() (n int) {
var l int
_ = l
@@ -3401,6 +3451,12 @@ func (m *PodSecurityPolicySpec) Size() (n int) {
n += 2 + l + sovGenerated(uint64(l))
}
}
if len(m.AllowedFlexVolumes) > 0 {
for _, e := range m.AllowedFlexVolumes {
l = e.Size()
n += 2 + l + sovGenerated(uint64(l))
}
}
return n
}
@@ -3605,6 +3661,16 @@ func sovGenerated(x uint64) (n int) {
func sozGenerated(x uint64) (n int) {
return sovGenerated(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (this *AllowedFlexVolume) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&AllowedFlexVolume{`,
`Driver:` + fmt.Sprintf("%v", this.Driver) + `,`,
`}`,
}, "")
return s
}
func (this *AllowedHostPath) String() string {
if this == nil {
return "nil"
@@ -4122,6 +4188,7 @@ func (this *PodSecurityPolicySpec) String() string {
`DefaultAllowPrivilegeEscalation:` + valueToStringGenerated(this.DefaultAllowPrivilegeEscalation) + `,`,
`AllowPrivilegeEscalation:` + valueToStringGenerated(this.AllowPrivilegeEscalation) + `,`,
`AllowedHostPaths:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AllowedHostPaths), "AllowedHostPath", "AllowedHostPath", 1), `&`, ``, 1) + `,`,
`AllowedFlexVolumes:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AllowedFlexVolumes), "AllowedFlexVolume", "AllowedFlexVolume", 1), `&`, ``, 1) + `,`,
`}`,
}, "")
return s
@@ -4316,6 +4383,85 @@ func valueToStringGenerated(v interface{}) string {
pv := reflect.Indirect(rv).Interface()
return fmt.Sprintf("*%v", pv)
}
func (m *AllowedFlexVolume) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: AllowedFlexVolume: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: AllowedFlexVolume: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Driver", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Driver = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthGenerated
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *AllowedHostPath) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@@ -10129,6 +10275,37 @@ func (m *PodSecurityPolicySpec) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 18:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field AllowedFlexVolumes", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowGenerated
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthGenerated
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.AllowedFlexVolumes = append(m.AllowedFlexVolumes, AllowedFlexVolume{})
if err := m.AllowedFlexVolumes[len(m.AllowedFlexVolumes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipGenerated(dAtA[iNdEx:])
@@ -12108,226 +12285,229 @@ func init() {
}
var fileDescriptorGenerated = []byte{
// 3523 bytes of a gzipped FileDescriptorProto
// 3571 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0xcd, 0x6f, 0x1c, 0x47,
0x76, 0x57, 0xcf, 0x0c, 0x39, 0xc3, 0x47, 0xf1, 0xab, 0x28, 0x93, 0x63, 0xca, 0xe2, 0xc8, 0x6d,
0x40, 0x91, 0x1c, 0x69, 0xc6, 0x92, 0x2d, 0x59, 0xb1, 0x10, 0x3b, 0x1c, 0x52, 0x1f, 0x74, 0xf8,
0xa5, 0x1a, 0x52, 0x71, 0x8c, 0xc8, 0x71, 0x73, 0xa6, 0x38, 0x6c, 0xb1, 0xa7, 0xbb, 0xdd, 0x5d,
0x4d, 0x73, 0x2e, 0x41, 0x0e, 0x41, 0x80, 0x00, 0x09, 0x92, 0x1c, 0x9c, 0x38, 0xb7, 0xf8, 0x92,
0x53, 0x82, 0xe4, 0x96, 0x1c, 0x0c, 0x03, 0x01, 0x1c, 0x40, 0x08, 0xbc, 0x80, 0x6f, 0xeb, 0x13,
0xb1, 0xa6, 0x4f, 0x8b, 0xfd, 0x07, 0x16, 0x3a, 0x2c, 0x16, 0x55, 0x5d, 0xfd, 0xdd, 0xcd, 0x99,
0xa1, 0x25, 0x62, 0xb1, 0xd8, 0x1b, 0xa7, 0xde, 0x7b, 0xbf, 0xf7, 0xea, 0xd5, 0xab, 0xf7, 0x5e,
0x57, 0x15, 0xe1, 0xde, 0xde, 0x6d, 0xbb, 0xaa, 0x1a, 0xb5, 0x3d, 0x67, 0x9b, 0x58, 0x3a, 0xa1,
0xc4, 0xae, 0xed, 0x13, 0xbd, 0x65, 0x58, 0x35, 0x41, 0x50, 0x4c, 0xb5, 0x46, 0x0e, 0x28, 0xd1,
0x6d, 0xd5, 0xd0, 0xed, 0xda, 0xfe, 0xf5, 0x6d, 0x42, 0x95, 0xeb, 0xb5, 0x36, 0xd1, 0x89, 0xa5,
0x50, 0xd2, 0xaa, 0x9a, 0x96, 0x41, 0x0d, 0x74, 0xc1, 0x65, 0xaf, 0x2a, 0xa6, 0x5a, 0x0d, 0xd8,
0xab, 0x82, 0x7d, 0xee, 0x5a, 0x5b, 0xa5, 0xbb, 0xce, 0x76, 0xb5, 0x69, 0x74, 0x6a, 0x6d, 0xa3,
0x6d, 0xd4, 0xb8, 0xd4, 0xb6, 0xb3, 0xc3, 0x7f, 0xf1, 0x1f, 0xfc, 0x2f, 0x17, 0x6d, 0x4e, 0x0e,
0x29, 0x6f, 0x1a, 0x16, 0xa9, 0xed, 0x27, 0x34, 0xce, 0x5d, 0x09, 0xf1, 0x98, 0x86, 0xa6, 0x36,
0xbb, 0x59, 0xc6, 0xcd, 0xbd, 0x15, 0xb0, 0x76, 0x94, 0xe6, 0xae, 0xaa, 0x13, 0xab, 0x5b, 0x33,
0xf7, 0xda, 0x5c, 0xd6, 0x22, 0xb6, 0xe1, 0x58, 0x4d, 0x32, 0x90, 0x94, 0x5d, 0xeb, 0x10, 0xaa,
0xa4, 0x99, 0x55, 0xcb, 0x92, 0xb2, 0x1c, 0x9d, 0xaa, 0x9d, 0xa4, 0x9a, 0x5b, 0xbd, 0x04, 0xec,
0xe6, 0x2e, 0xe9, 0x28, 0x09, 0xb9, 0x37, 0xb3, 0xe4, 0x1c, 0xaa, 0x6a, 0x35, 0x55, 0xa7, 0x36,
0xb5, 0xe2, 0x42, 0xf2, 0x5d, 0x98, 0x58, 0xd0, 0x34, 0xe3, 0x53, 0xd2, 0x7a, 0x60, 0xd8, 0x74,
0x43, 0xa1, 0xbb, 0xe8, 0x06, 0x80, 0xa9, 0xd0, 0xdd, 0x0d, 0x8b, 0xec, 0xa8, 0x07, 0x65, 0xe9,
0xa2, 0x74, 0x79, 0xa4, 0x8e, 0x9e, 0x1e, 0x56, 0xce, 0x1c, 0x1d, 0x56, 0x60, 0xc3, 0xa7, 0xe0,
0x10, 0x97, 0xfc, 0x2f, 0x12, 0xbc, 0xbc, 0xe8, 0xd8, 0xd4, 0xe8, 0xac, 0x12, 0x6a, 0xa9, 0xcd,
0x45, 0xc7, 0xb2, 0x88, 0x4e, 0x1b, 0x54, 0xa1, 0x8e, 0x8d, 0x2e, 0x42, 0x41, 0x57, 0x3a, 0x44,
0x60, 0x9d, 0x15, 0x58, 0x85, 0x35, 0xa5, 0x43, 0x30, 0xa7, 0xa0, 0x0f, 0x61, 0x68, 0x5f, 0xd1,
0x1c, 0x52, 0xce, 0x5d, 0x94, 0x2e, 0x8f, 0xde, 0xa8, 0x56, 0x83, 0xe8, 0xf1, 0xe7, 0x52, 0x35,
0xf7, 0xda, 0x3c, 0x9c, 0xbc, 0x05, 0xaa, 0x3e, 0x74, 0x14, 0x9d, 0xaa, 0xb4, 0x5b, 0x3f, 0x27,
0x20, 0xcf, 0x0a, 0xbd, 0x8f, 0x18, 0x16, 0x76, 0x21, 0xe5, 0xbf, 0x80, 0x0b, 0x99, 0xa6, 0xad,
0xa8, 0x36, 0x45, 0x8f, 0x61, 0x48, 0xa5, 0xa4, 0x63, 0x97, 0xa5, 0x8b, 0xf9, 0xcb, 0xa3, 0x37,
0x6e, 0x57, 0x8f, 0x0d, 0xdd, 0x6a, 0x26, 0x58, 0x7d, 0x4c, 0x98, 0x31, 0xb4, 0xcc, 0xe0, 0xb0,
0x8b, 0x2a, 0xff, 0xa3, 0x04, 0x28, 0x2c, 0xb3, 0xa9, 0x58, 0x6d, 0x42, 0xfb, 0x70, 0xca, 0x9f,
0xfe, 0x38, 0xa7, 0x4c, 0x0b, 0xc8, 0x51, 0x57, 0x61, 0xc4, 0x27, 0x26, 0xcc, 0x24, 0x4d, 0xe2,
0xce, 0x78, 0x14, 0x75, 0xc6, 0xf5, 0x01, 0x9c, 0xe1, 0xa2, 0x64, 0x78, 0xe1, 0xb3, 0x1c, 0x8c,
0x2c, 0x29, 0xa4, 0x63, 0xe8, 0x0d, 0x42, 0xd1, 0xc7, 0x50, 0x62, 0xfb, 0xa5, 0xa5, 0x50, 0x85,
0x3b, 0x60, 0xf4, 0xc6, 0x1b, 0xc7, 0xcd, 0xce, 0xae, 0x32, 0xee, 0xea, 0xfe, 0xf5, 0xea, 0xfa,
0xf6, 0x13, 0xd2, 0xa4, 0xab, 0x84, 0x2a, 0x41, 0x4c, 0x06, 0x63, 0xd8, 0x47, 0x45, 0x6b, 0x50,
0xb0, 0x4d, 0xd2, 0x14, 0xbe, 0xbb, 0xda, 0x63, 0x1a, 0xbe, 0x65, 0x0d, 0x93, 0x34, 0x83, 0xc5,
0x60, 0xbf, 0x30, 0xc7, 0x41, 0x8f, 0x60, 0xd8, 0xe6, 0xab, 0x5c, 0xce, 0x27, 0x56, 0xe3, 0x78,
0x44, 0x37, 0x36, 0xc6, 0x05, 0xe6, 0xb0, 0xfb, 0x1b, 0x0b, 0x34, 0xf9, 0xe7, 0x39, 0x40, 0x3e,
0xef, 0xa2, 0xa1, 0xb7, 0x54, 0xaa, 0x1a, 0x3a, 0x7a, 0x07, 0x0a, 0xb4, 0x6b, 0x7a, 0xd1, 0x71,
0xc9, 0x33, 0x68, 0xb3, 0x6b, 0x92, 0x67, 0x87, 0x95, 0x99, 0xa4, 0x04, 0xa3, 0x60, 0x2e, 0x83,
0x56, 0x7c, 0x53, 0x73, 0x5c, 0xfa, 0xad, 0xa8, 0xea, 0x67, 0x87, 0x95, 0x94, 0x74, 0x5a, 0xf5,
0x91, 0xa2, 0x06, 0xa2, 0x7d, 0x40, 0x9a, 0x62, 0xd3, 0x4d, 0x4b, 0xd1, 0x6d, 0x57, 0x93, 0xda,
0x21, 0xc2, 0x09, 0xaf, 0xf7, 0xb7, 0x68, 0x4c, 0xa2, 0x3e, 0x27, 0xac, 0x40, 0x2b, 0x09, 0x34,
0x9c, 0xa2, 0x01, 0x5d, 0x82, 0x61, 0x8b, 0x28, 0xb6, 0xa1, 0x97, 0x0b, 0x7c, 0x16, 0xbe, 0x03,
0x31, 0x1f, 0xc5, 0x82, 0x8a, 0xae, 0x40, 0xb1, 0x43, 0x6c, 0x5b, 0x69, 0x93, 0xf2, 0x10, 0x67,
0x9c, 0x10, 0x8c, 0xc5, 0x55, 0x77, 0x18, 0x7b, 0x74, 0xf9, 0x4b, 0x09, 0xc6, 0x7c, 0xcf, 0xf1,
0x68, 0xff, 0xb3, 0x44, 0x1c, 0x56, 0xfb, 0x9b, 0x12, 0x93, 0xe6, 0x51, 0x38, 0x29, 0xb4, 0x95,
0xbc, 0x91, 0x50, 0x0c, 0xae, 0x7a, 0x7b, 0x29, 0xc7, 0xf7, 0xd2, 0xe5, 0x7e, 0x43, 0x26, 0x63,
0x0b, 0xfd, 0x53, 0x21, 0x64, 0x3e, 0x0b, 0x4d, 0xf4, 0x18, 0x4a, 0x36, 0xd1, 0x48, 0x93, 0x1a,
0x96, 0x30, 0xff, 0xcd, 0x3e, 0xcd, 0x57, 0xb6, 0x89, 0xd6, 0x10, 0xa2, 0xf5, 0xb3, 0xcc, 0x7e,
0xef, 0x17, 0xf6, 0x21, 0xd1, 0x43, 0x28, 0x51, 0xd2, 0x31, 0x35, 0x85, 0x7a, 0x39, 0xe8, 0xb5,
0xf0, 0x14, 0x58, 0xe4, 0x30, 0xb0, 0x0d, 0xa3, 0xb5, 0x29, 0xd8, 0xf8, 0xf6, 0xf1, 0x5d, 0xe2,
0x8d, 0x62, 0x1f, 0x06, 0xed, 0xc3, 0xb8, 0x63, 0xb6, 0x18, 0x27, 0x65, 0x65, 0xa8, 0xdd, 0x15,
0x91, 0x74, 0xab, 0x5f, 0xdf, 0x6c, 0x45, 0xa4, 0xeb, 0x33, 0x42, 0xd7, 0x78, 0x74, 0x1c, 0xc7,
0xb4, 0xa0, 0x05, 0x98, 0xe8, 0xa8, 0x3a, 0x26, 0x4a, 0xab, 0xdb, 0x20, 0x4d, 0x43, 0x6f, 0xd9,
0x3c, 0xac, 0x86, 0xea, 0xb3, 0x02, 0x60, 0x62, 0x35, 0x4a, 0xc6, 0x71, 0x7e, 0xf4, 0x3e, 0x20,
0x6f, 0x1a, 0xf7, 0xdd, 0x2a, 0xaa, 0x1a, 0x3a, 0x8f, 0xb9, 0x7c, 0x10, 0xdc, 0x9b, 0x09, 0x0e,
0x9c, 0x22, 0x85, 0x56, 0xe0, 0x9c, 0x45, 0xf6, 0x55, 0x36, 0xc7, 0x07, 0xaa, 0x4d, 0x0d, 0xab,
0xbb, 0xa2, 0x76, 0x54, 0x5a, 0x1e, 0xe6, 0x36, 0x95, 0x8f, 0x0e, 0x2b, 0xe7, 0x70, 0x0a, 0x1d,
0xa7, 0x4a, 0xc9, 0xff, 0x3c, 0x0c, 0x13, 0xb1, 0x7c, 0x83, 0x1e, 0xc1, 0x4c, 0xd3, 0x2d, 0x4e,
0x6b, 0x4e, 0x67, 0x9b, 0x58, 0x8d, 0xe6, 0x2e, 0x69, 0x39, 0x1a, 0x69, 0xf1, 0x40, 0x19, 0xaa,
0xcf, 0x0b, 0x8b, 0x67, 0x16, 0x53, 0xb9, 0x70, 0x86, 0x34, 0xf3, 0x82, 0xce, 0x87, 0x56, 0x55,
0xdb, 0xf6, 0x31, 0x73, 0x1c, 0xd3, 0xf7, 0xc2, 0x5a, 0x82, 0x03, 0xa7, 0x48, 0x31, 0x1b, 0x5b,
0xc4, 0x56, 0x2d, 0xd2, 0x8a, 0xdb, 0x98, 0x8f, 0xda, 0xb8, 0x94, 0xca, 0x85, 0x33, 0xa4, 0xd1,
0x4d, 0x18, 0x75, 0xb5, 0xf1, 0xf5, 0x13, 0x0b, 0xed, 0x97, 0xc3, 0xb5, 0x80, 0x84, 0xc3, 0x7c,
0x6c, 0x6a, 0xc6, 0xb6, 0x4d, 0xac, 0x7d, 0xd2, 0xca, 0x5e, 0xe0, 0xf5, 0x04, 0x07, 0x4e, 0x91,
0x62, 0x53, 0x73, 0x23, 0x30, 0x31, 0xb5, 0xe1, 0xe8, 0xd4, 0xb6, 0x52, 0xb9, 0x70, 0x86, 0x34,
0x8b, 0x63, 0xd7, 0xe4, 0x85, 0x7d, 0x45, 0xd5, 0x94, 0x6d, 0x8d, 0x94, 0x8b, 0xd1, 0x38, 0x5e,
0x8b, 0x92, 0x71, 0x9c, 0x1f, 0xdd, 0x87, 0x29, 0x77, 0x68, 0x4b, 0x57, 0x7c, 0x90, 0x12, 0x07,
0x79, 0x59, 0x80, 0x4c, 0xad, 0xc5, 0x19, 0x70, 0x52, 0x06, 0xbd, 0x03, 0xe3, 0x4d, 0x43, 0xd3,
0x78, 0x3c, 0x2e, 0x1a, 0x8e, 0x4e, 0xcb, 0x23, 0x1c, 0x05, 0xb1, 0xfd, 0xb8, 0x18, 0xa1, 0xe0,
0x18, 0x27, 0x22, 0x00, 0x4d, 0xaf, 0xe0, 0xd8, 0x65, 0xe8, 0xab, 0xd7, 0x48, 0x16, 0xbd, 0xa0,
0x07, 0xf0, 0x87, 0x6c, 0x1c, 0x02, 0x96, 0xff, 0x5f, 0x82, 0xd9, 0x8c, 0xd4, 0x81, 0xde, 0x8b,
0x94, 0xd8, 0xdf, 0x8f, 0x95, 0xd8, 0xf3, 0x19, 0x62, 0xa1, 0x3a, 0xab, 0xc3, 0x98, 0xc5, 0x66,
0xa5, 0xb7, 0x5d, 0x16, 0x91, 0x23, 0x6f, 0xf6, 0x98, 0x06, 0x0e, 0xcb, 0x04, 0x39, 0x7f, 0xea,
0xe8, 0xb0, 0x32, 0x16, 0xa1, 0xe1, 0x28, 0xbc, 0xfc, 0x79, 0x0e, 0x60, 0x89, 0x98, 0x9a, 0xd1,
0xed, 0x10, 0xfd, 0x34, 0x7a, 0xa8, 0xf5, 0x48, 0x0f, 0x75, 0xad, 0xd7, 0xf2, 0xf8, 0xa6, 0x65,
0x36, 0x51, 0x7f, 0x12, 0x6b, 0xa2, 0x6a, 0xfd, 0x43, 0x1e, 0xdf, 0x45, 0xfd, 0x34, 0x0f, 0xd3,
0x01, 0x73, 0xd0, 0x46, 0xdd, 0x89, 0xac, 0xf1, 0xef, 0xc5, 0xd6, 0x78, 0x36, 0x45, 0xe4, 0x85,
0xf5, 0x51, 0xcf, 0xbf, 0x9f, 0x41, 0x4f, 0x60, 0x9c, 0x35, 0x4e, 0x6e, 0x78, 0xf0, 0xb6, 0x6c,
0x78, 0xe0, 0xb6, 0xcc, 0x2f, 0xa0, 0x2b, 0x11, 0x24, 0x1c, 0x43, 0xce, 0x68, 0x03, 0x8b, 0x2f,
0xba, 0x0d, 0x94, 0xbf, 0x92, 0x60, 0x3c, 0x58, 0xa6, 0x53, 0x68, 0xda, 0xd6, 0xa2, 0x4d, 0xdb,
0x95, 0xbe, 0x43, 0x34, 0xa3, 0x6b, 0xfb, 0x25, 0x6b, 0xf0, 0x7d, 0x26, 0xb6, 0xc1, 0xb7, 0x95,
0xe6, 0x5e, 0x1f, 0x9f, 0x7f, 0x9f, 0x49, 0x80, 0x44, 0x15, 0x58, 0xd0, 0x75, 0x83, 0x2a, 0x6e,
0xae, 0x74, 0xcd, 0x5a, 0xee, 0xdb, 0x2c, 0x4f, 0x63, 0x75, 0x2b, 0x81, 0x75, 0x57, 0xa7, 0x56,
0x37, 0x58, 0x91, 0x24, 0x03, 0x4e, 0x31, 0x00, 0x29, 0x00, 0x96, 0xc0, 0xdc, 0x34, 0xc4, 0x46,
0xbe, 0xd6, 0x47, 0xce, 0x63, 0x02, 0x8b, 0x86, 0xbe, 0xa3, 0xb6, 0x83, 0xb4, 0x83, 0x7d, 0x20,
0x1c, 0x02, 0x9d, 0xbb, 0x0b, 0xb3, 0x19, 0xd6, 0xa2, 0x49, 0xc8, 0xef, 0x91, 0xae, 0xeb, 0x36,
0xcc, 0xfe, 0x44, 0xe7, 0xc2, 0x9f, 0xc9, 0x23, 0xe2, 0x0b, 0xf7, 0x9d, 0xdc, 0x6d, 0x49, 0xfe,
0x72, 0x28, 0x1c, 0x3b, 0xbc, 0x63, 0xbe, 0x0c, 0x25, 0x8b, 0x98, 0x9a, 0xda, 0x54, 0x6c, 0xd1,
0x08, 0xf1, 0xe6, 0x17, 0x8b, 0x31, 0xec, 0x53, 0x23, 0xbd, 0x75, 0xee, 0xc5, 0xf6, 0xd6, 0xf9,
0xe7, 0xd3, 0x5b, 0xff, 0x39, 0x94, 0x6c, 0xaf, 0xab, 0x2e, 0x70, 0xc8, 0xeb, 0x03, 0xe4, 0x57,
0xd1, 0x50, 0xfb, 0x0a, 0xfc, 0x56, 0xda, 0x07, 0x4d, 0x6b, 0xa2, 0x87, 0x06, 0x6c, 0xa2, 0x9f,
0x6b, 0xe3, 0xcb, 0x72, 0xaa, 0xa9, 0x38, 0x36, 0x69, 0xf1, 0x44, 0x54, 0x0a, 0x72, 0xea, 0x06,
0x1f, 0xc5, 0x82, 0x8a, 0x1e, 0x47, 0x42, 0xb6, 0x74, 0x92, 0x90, 0x1d, 0xcf, 0x0e, 0x57, 0xb4,
0x05, 0xb3, 0xa6, 0x65, 0xb4, 0x2d, 0x62, 0xdb, 0x4b, 0x44, 0x69, 0x69, 0xaa, 0x4e, 0x3c, 0xff,
0xb8, 0x1d, 0xd1, 0xf9, 0xa3, 0xc3, 0xca, 0xec, 0x46, 0x3a, 0x0b, 0xce, 0x92, 0x95, 0x9f, 0x16,
0x60, 0x32, 0x5e, 0x01, 0x33, 0x9a, 0x54, 0xe9, 0x44, 0x4d, 0xea, 0xd5, 0xd0, 0x66, 0x70, 0x3b,
0x78, 0x7f, 0xf5, 0x53, 0x36, 0xc4, 0x02, 0x4c, 0x88, 0x6c, 0xe0, 0x11, 0x45, 0x9b, 0xee, 0xaf,
0xfe, 0x56, 0x94, 0x8c, 0xe3, 0xfc, 0xac, 0xf5, 0x0c, 0x3a, 0x4a, 0x0f, 0xa4, 0x10, 0x6d, 0x3d,
0x17, 0xe2, 0x0c, 0x38, 0x29, 0x83, 0x56, 0x61, 0xda, 0xd1, 0x93, 0x50, 0x6e, 0x34, 0x9e, 0x17,
0x50, 0xd3, 0x5b, 0x49, 0x16, 0x9c, 0x26, 0x87, 0x76, 0x22, 0xdd, 0xe8, 0x30, 0xcf, 0xb0, 0x37,
0xfa, 0xde, 0x3b, 0x7d, 0xb7, 0xa3, 0xe8, 0x0e, 0x8c, 0x59, 0xfc, 0xbb, 0xc3, 0x33, 0xd8, 0xed,
0xdd, 0x5f, 0x12, 0x62, 0x63, 0x38, 0x4c, 0xc4, 0x51, 0xde, 0x94, 0x76, 0xbb, 0xd4, 0x6f, 0xbb,
0x2d, 0xff, 0xaf, 0x14, 0x2e, 0x42, 0x7e, 0x0b, 0xdc, 0xeb, 0x94, 0x29, 0x21, 0x11, 0xea, 0x8e,
0x8c, 0xf4, 0xee, 0xf7, 0xd6, 0x40, 0xdd, 0x6f, 0x50, 0x3c, 0x7b, 0xb7, 0xbf, 0x5f, 0x48, 0x30,
0x73, 0xaf, 0x71, 0xdf, 0x32, 0x1c, 0xd3, 0x33, 0x67, 0xdd, 0x74, 0xfd, 0xfa, 0x36, 0x14, 0x2c,
0x47, 0xf3, 0xe6, 0xf1, 0x9a, 0x37, 0x0f, 0xec, 0x68, 0x6c, 0x1e, 0xd3, 0x31, 0x29, 0x77, 0x12,
0x4c, 0x00, 0xad, 0xc1, 0xb0, 0xa5, 0xe8, 0x6d, 0xe2, 0x95, 0xd5, 0x4b, 0x3d, 0xac, 0x5f, 0x5e,
0xc2, 0x8c, 0x3d, 0xd4, 0xbc, 0x71, 0x69, 0x2c, 0x50, 0xe4, 0xbf, 0x93, 0x60, 0xe2, 0xc1, 0xe6,
0xe6, 0xc6, 0xb2, 0xce, 0x77, 0x34, 0x3f, 0x4f, 0xbf, 0x08, 0x05, 0x53, 0xa1, 0xbb, 0xf1, 0x4a,
0xcf, 0x68, 0x98, 0x53, 0xd0, 0x07, 0x50, 0x64, 0x99, 0x84, 0xe8, 0xad, 0x3e, 0x5b, 0x6d, 0x01,
0x5f, 0x77, 0x85, 0x82, 0x0e, 0x51, 0x0c, 0x60, 0x0f, 0x4e, 0xde, 0x83, 0x73, 0x21, 0x73, 0x98,
0x3f, 0xf8, 0x31, 0x30, 0x6a, 0xc0, 0x10, 0xd3, 0xec, 0x9d, 0xf2, 0xf6, 0x3a, 0xcc, 0x8c, 0x4d,
0x29, 0xe8, 0x74, 0xd8, 0x2f, 0x1b, 0xbb, 0x58, 0xf2, 0x2a, 0x8c, 0xf1, 0x4b, 0x04, 0xc3, 0xa2,
0xdc, 0x2d, 0xe8, 0x02, 0xe4, 0x3b, 0xaa, 0x2e, 0xea, 0xec, 0xa8, 0x90, 0xc9, 0xb3, 0x1a, 0xc1,
0xc6, 0x39, 0x59, 0x39, 0x10, 0x99, 0x27, 0x20, 0x2b, 0x07, 0x98, 0x8d, 0xcb, 0xf7, 0xa1, 0x28,
0xdc, 0x1d, 0x06, 0xca, 0x1f, 0x0f, 0x94, 0x4f, 0x01, 0x5a, 0x87, 0xe2, 0xf2, 0x46, 0x5d, 0x33,
0xdc, 0xae, 0xab, 0xa9, 0xb6, 0xac, 0xf8, 0x5a, 0x2c, 0x2e, 0x2f, 0x61, 0xcc, 0x29, 0x48, 0x86,
0x61, 0x72, 0xd0, 0x24, 0x26, 0xe5, 0x11, 0x31, 0x52, 0x07, 0xb6, 0xca, 0x77, 0xf9, 0x08, 0x16,
0x14, 0xf9, 0xef, 0x73, 0x50, 0x14, 0xee, 0x38, 0x85, 0xaf, 0xb0, 0x95, 0xc8, 0x57, 0xd8, 0xeb,
0xfd, 0x85, 0x46, 0xe6, 0x27, 0xd8, 0x66, 0xec, 0x13, 0xec, 0x6a, 0x9f, 0x78, 0xc7, 0x7f, 0x7f,
0xfd, 0x97, 0x04, 0xe3, 0xd1, 0xa0, 0x44, 0x37, 0x61, 0x94, 0x15, 0x1c, 0xb5, 0x49, 0xd6, 0x82,
0x3e, 0xd7, 0x3f, 0x84, 0x69, 0x04, 0x24, 0x1c, 0xe6, 0x43, 0x6d, 0x5f, 0x8c, 0xc5, 0x91, 0x98,
0x74, 0xb6, 0x4b, 0x1d, 0xaa, 0x6a, 0x55, 0xf7, 0x6e, 0xab, 0xba, 0xac, 0xd3, 0x75, 0xab, 0x41,
0x2d, 0x55, 0x6f, 0x27, 0x14, 0xf1, 0xa0, 0x0c, 0x23, 0xcb, 0xff, 0x23, 0xc1, 0xa8, 0x30, 0xf9,
0x14, 0xbe, 0x2a, 0xfe, 0x38, 0xfa, 0x55, 0x71, 0xa9, 0xcf, 0x0d, 0x9e, 0xfe, 0x49, 0xf1, 0x6f,
0x81, 0xe9, 0x6c, 0x4b, 0xb3, 0xa8, 0xde, 0x35, 0x6c, 0x1a, 0x8f, 0x6a, 0xb6, 0x19, 0x31, 0xa7,
0x20, 0x07, 0x26, 0xd5, 0x58, 0x0e, 0x10, 0xae, 0xad, 0xf5, 0x67, 0x89, 0x2f, 0x56, 0x2f, 0x0b,
0xf8, 0xc9, 0x38, 0x05, 0x27, 0x54, 0xc8, 0x04, 0x12, 0x5c, 0xe8, 0x21, 0x14, 0x76, 0x29, 0x35,
0x53, 0xce, 0xab, 0x7b, 0x64, 0x9e, 0xc0, 0x84, 0x12, 0x9f, 0xdd, 0xe6, 0xe6, 0x06, 0xe6, 0x50,
0xf2, 0xaf, 0x02, 0x7f, 0x34, 0xdc, 0x18, 0xf7, 0xf3, 0xa9, 0x74, 0x92, 0x7c, 0x3a, 0x9a, 0x96,
0x4b, 0xd1, 0x03, 0xc8, 0x53, 0xad, 0xdf, 0xcf, 0x42, 0x81, 0xb8, 0xb9, 0xd2, 0x08, 0x12, 0xd2,
0xe6, 0x4a, 0x03, 0x33, 0x08, 0xb4, 0x0e, 0x43, 0xac, 0xfa, 0xb0, 0x2d, 0x98, 0xef, 0x7f, 0x4b,
0xb3, 0xf9, 0x07, 0x01, 0xc1, 0x7e, 0xd9, 0xd8, 0xc5, 0x91, 0x3f, 0x81, 0xb1, 0xc8, 0x3e, 0x45,
0x1f, 0xc3, 0x59, 0xcd, 0x50, 0x5a, 0x75, 0x45, 0x53, 0xf4, 0x26, 0xf1, 0x2e, 0x07, 0x2e, 0xa5,
0x7d, 0x61, 0xac, 0x84, 0xf8, 0xc4, 0x2e, 0xf7, 0xaf, 0x53, 0xc3, 0x34, 0x1c, 0x41, 0x94, 0x15,
0x80, 0x60, 0x8e, 0xa8, 0x02, 0x43, 0x2c, 0xce, 0xdc, 0x7a, 0x32, 0x52, 0x1f, 0x61, 0x16, 0xb2,
0xf0, 0xb3, 0xb1, 0x3b, 0x8e, 0x6e, 0x00, 0xd8, 0xa4, 0x69, 0x11, 0xca, 0x93, 0x41, 0x2e, 0x7a,
0xa9, 0xdc, 0xf0, 0x29, 0x38, 0xc4, 0x25, 0xff, 0x9f, 0x04, 0x63, 0x6b, 0x84, 0x7e, 0x6a, 0x58,
0x7b, 0x1b, 0xfc, 0x3e, 0xff, 0x14, 0x92, 0x2d, 0x8e, 0x24, 0xdb, 0x37, 0x7a, 0xac, 0x4c, 0xc4,
0xba, 0xac, 0x94, 0x2b, 0x7f, 0x25, 0xc1, 0x6c, 0x84, 0xf3, 0x6e, 0xb0, 0x75, 0xb7, 0x60, 0xc8,
0x34, 0x2c, 0xea, 0x15, 0xe2, 0x81, 0x14, 0xb2, 0x34, 0x16, 0x2a, 0xc5, 0x0c, 0x06, 0xbb, 0x68,
0x68, 0x05, 0x72, 0xd4, 0x10, 0xa1, 0x3a, 0x18, 0x26, 0x21, 0x56, 0x1d, 0x04, 0x66, 0x6e, 0xd3,
0xc0, 0x39, 0x6a, 0xb0, 0x85, 0x28, 0x47, 0xb8, 0xc2, 0xc9, 0xe7, 0x05, 0xcd, 0x00, 0x43, 0x61,
0xc7, 0x32, 0x3a, 0x27, 0x9e, 0x83, 0xbf, 0x10, 0xf7, 0x2c, 0xa3, 0x83, 0x39, 0x96, 0xfc, 0xb5,
0x04, 0x53, 0x11, 0xce, 0x53, 0x48, 0xfc, 0x0f, 0xa3, 0x89, 0xff, 0xea, 0x20, 0x13, 0xc9, 0x48,
0xff, 0x5f, 0xe7, 0x62, 0xd3, 0x60, 0x13, 0x46, 0x3b, 0x30, 0x6a, 0x1a, 0xad, 0xc6, 0x73, 0xb8,
0x0e, 0x9c, 0x60, 0x75, 0x73, 0x23, 0xc0, 0xc2, 0x61, 0x60, 0x74, 0x00, 0x53, 0xba, 0xd2, 0x21,
0xb6, 0xa9, 0x34, 0x49, 0xe3, 0x39, 0x1c, 0x90, 0xbc, 0xc4, 0xef, 0x1b, 0xe2, 0x88, 0x38, 0xa9,
0x04, 0xad, 0x42, 0x51, 0x35, 0x79, 0x1f, 0x27, 0x7a, 0x97, 0x9e, 0x55, 0xd4, 0xed, 0xfa, 0xdc,
0x7c, 0x2e, 0x7e, 0x60, 0x0f, 0x43, 0xfe, 0xf7, 0x78, 0x34, 0xb0, 0xf8, 0x43, 0xf7, 0xa1, 0xc4,
0x5f, 0xc6, 0x34, 0x0d, 0xcd, 0xbb, 0x19, 0x60, 0x2b, 0xbb, 0x21, 0xc6, 0x9e, 0x1d, 0x56, 0xce,
0xa7, 0x1c, 0xfa, 0x7a, 0x64, 0xec, 0x0b, 0xa3, 0x35, 0x28, 0x98, 0x3f, 0xa6, 0x83, 0xe1, 0x45,
0x8e, 0xb7, 0x2d, 0x1c, 0x47, 0xfe, 0xab, 0x7c, 0xcc, 0x5c, 0x5e, 0xea, 0x9e, 0x3c, 0xb7, 0x55,
0xf7, 0x3b, 0xa6, 0xcc, 0x95, 0xdf, 0x86, 0xa2, 0xa8, 0xf0, 0x22, 0x98, 0xdf, 0x1e, 0x24, 0x98,
0xc3, 0x55, 0xcc, 0xff, 0x60, 0xf1, 0x06, 0x3d, 0x60, 0xf4, 0x11, 0x0c, 0x13, 0x57, 0x85, 0x5b,
0x1b, 0x6f, 0x0d, 0xa2, 0x22, 0xc8, 0xab, 0x41, 0xa3, 0x2a, 0xc6, 0x04, 0x2a, 0x7a, 0x8f, 0xf9,
0x8b, 0xf1, 0xb2, 0x8f, 0x40, 0xbb, 0x5c, 0xe0, 0xe5, 0xea, 0x82, 0x3b, 0x6d, 0x7f, 0xf8, 0xd9,
0x61, 0x05, 0x82, 0x9f, 0x38, 0x2c, 0x21, 0xff, 0x44, 0x82, 0x29, 0xee, 0xa1, 0xa6, 0x63, 0xa9,
0xb4, 0x7b, 0x6a, 0x85, 0xe9, 0x51, 0xa4, 0x30, 0xbd, 0xd5, 0xc3, 0x2d, 0x09, 0x0b, 0x33, 0x8b,
0xd3, 0x37, 0x12, 0xbc, 0x94, 0xe0, 0x3e, 0x85, 0xbc, 0xb8, 0x15, 0xcd, 0x8b, 0x6f, 0x0c, 0x3a,
0xa1, 0x8c, 0xdc, 0xf8, 0xf9, 0xd9, 0x94, 0xe9, 0xf0, 0x9d, 0x72, 0x03, 0xc0, 0xb4, 0xd4, 0x7d,
0x55, 0x23, 0x6d, 0x71, 0x09, 0x5e, 0x0a, 0x3d, 0x6b, 0xf3, 0x29, 0x38, 0xc4, 0x85, 0x6c, 0x98,
0x69, 0x91, 0x1d, 0xc5, 0xd1, 0xe8, 0x42, 0xab, 0xb5, 0xa8, 0x98, 0xca, 0xb6, 0xaa, 0xa9, 0x54,
0x15, 0xc7, 0x05, 0x23, 0xf5, 0x3b, 0xee, 0xe5, 0x74, 0x1a, 0xc7, 0xb3, 0xc3, 0xca, 0x85, 0xb4,
0xdb, 0x21, 0x8f, 0xa5, 0x8b, 0x33, 0xa0, 0x51, 0x17, 0xca, 0x16, 0xf9, 0xc4, 0x51, 0x2d, 0xd2,
0x5a, 0xb2, 0x0c, 0x33, 0xa2, 0x36, 0xcf, 0xd5, 0xfe, 0xe1, 0xd1, 0x61, 0xa5, 0x8c, 0x33, 0x78,
0x7a, 0x2b, 0xce, 0x84, 0x47, 0x4f, 0x60, 0x5a, 0x71, 0x5f, 0x03, 0x46, 0xb4, 0xba, 0xbb, 0xe4,
0xf6, 0xd1, 0x61, 0x65, 0x7a, 0x21, 0x49, 0xee, 0xad, 0x30, 0x0d, 0x14, 0xd5, 0xa0, 0xb8, 0x6f,
0x68, 0x4e, 0x87, 0xd8, 0xe5, 0x21, 0x8e, 0xcf, 0x0a, 0x41, 0xf1, 0x91, 0x3b, 0xf4, 0xec, 0xb0,
0x32, 0x7c, 0xaf, 0xc1, 0x77, 0x9f, 0xc7, 0xc5, 0x3e, 0x28, 0x59, 0x2f, 0x29, 0x76, 0x3c, 0x3f,
0x31, 0x2e, 0x05, 0x59, 0xeb, 0x41, 0x40, 0xc2, 0x61, 0x3e, 0xf4, 0x18, 0x46, 0x76, 0xc5, 0xa9,
0x84, 0x5d, 0x2e, 0xf6, 0x55, 0x84, 0x23, 0xa7, 0x18, 0xf5, 0x29, 0xa1, 0x62, 0xc4, 0x1b, 0xb6,
0x71, 0x80, 0x88, 0xae, 0x40, 0x91, 0xff, 0x58, 0x5e, 0xe2, 0xc7, 0x71, 0xa5, 0x20, 0xb7, 0x3d,
0x70, 0x87, 0xb1, 0x47, 0xf7, 0x58, 0x97, 0x37, 0x16, 0xf9, 0xb1, 0x70, 0x8c, 0x75, 0x79, 0x63,
0x11, 0x7b, 0x74, 0xf4, 0x31, 0x14, 0x6d, 0xb2, 0xa2, 0xea, 0xce, 0x41, 0x19, 0xfa, 0xba, 0x54,
0x6e, 0xdc, 0xe5, 0xdc, 0xb1, 0x83, 0xb1, 0x40, 0x83, 0xa0, 0x63, 0x0f, 0x16, 0xed, 0xc2, 0x88,
0xe5, 0xe8, 0x0b, 0xf6, 0x96, 0x4d, 0xac, 0xf2, 0x28, 0xd7, 0xd1, 0x2b, 0x9d, 0x63, 0x8f, 0x3f,
0xae, 0xc5, 0xf7, 0x90, 0xcf, 0x81, 0x03, 0x70, 0xf4, 0xb7, 0x12, 0x20, 0xdb, 0x31, 0x4d, 0x8d,
0x74, 0x88, 0x4e, 0x15, 0x8d, 0x9f, 0xc5, 0xd9, 0xe5, 0xb3, 0x5c, 0xe7, 0x1f, 0xf5, 0x9a, 0x57,
0x42, 0x30, 0xae, 0xdc, 0x3f, 0xf4, 0x4e, 0xb2, 0xe2, 0x14, 0xbd, 0xcc, 0xb5, 0x3b, 0x36, 0xff,
0xbb, 0x3c, 0xd6, 0x97, 0x6b, 0xd3, 0xcf, 0x1c, 0x03, 0xd7, 0x0a, 0x3a, 0xf6, 0x60, 0xd1, 0x23,
0x98, 0xb1, 0x88, 0xd2, 0x5a, 0xd7, 0xb5, 0x2e, 0x36, 0x0c, 0x7a, 0x4f, 0xd5, 0x88, 0xdd, 0xb5,
0x29, 0xe9, 0x94, 0xc7, 0xf9, 0xb2, 0xfb, 0x6f, 0x3f, 0x70, 0x2a, 0x17, 0xce, 0x90, 0x46, 0x1d,
0xa8, 0x78, 0x29, 0x83, 0xed, 0x27, 0x3f, 0x67, 0xdd, 0xb5, 0x9b, 0x8a, 0xe6, 0xde, 0x03, 0x4c,
0x70, 0x05, 0xaf, 0x1d, 0x1d, 0x56, 0x2a, 0x4b, 0xc7, 0xb3, 0xe2, 0x5e, 0x58, 0xe8, 0x03, 0x28,
0x2b, 0x59, 0x7a, 0x26, 0xb9, 0x9e, 0x57, 0x58, 0x1e, 0xca, 0x54, 0x90, 0x29, 0x8d, 0x28, 0x4c,
0x2a, 0xd1, 0x47, 0xc7, 0x76, 0x79, 0xaa, 0xaf, 0x83, 0xc8, 0xd8, 0x5b, 0xe5, 0xe0, 0x30, 0x22,
0x46, 0xb0, 0x71, 0x42, 0x03, 0x7f, 0x3e, 0x21, 0x0e, 0xd3, 0x4f, 0xe7, 0x09, 0xea, 0x60, 0xcf,
0x27, 0x02, 0xd3, 0x9e, 0xdb, 0xf3, 0x89, 0x10, 0xe4, 0xf1, 0xc7, 0x77, 0xbf, 0xc8, 0xc1, 0x74,
0xc0, 0xdc, 0xf7, 0xf3, 0x89, 0x14, 0x91, 0xdf, 0x3d, 0x43, 0xed, 0xfd, 0x0c, 0xf5, 0x2b, 0x09,
0xc6, 0x03, 0xd7, 0xfd, 0xe6, 0x3d, 0x69, 0x08, 0x6c, 0xcb, 0x68, 0xb2, 0xfe, 0x33, 0x17, 0x9e,
0xc0, 0x6f, 0xfd, 0xbd, 0xfa, 0x8f, 0x7f, 0x3b, 0x2a, 0x7f, 0x93, 0x87, 0xc9, 0xf8, 0x6e, 0x8c,
0x5c, 0xbf, 0x4a, 0x3d, 0xaf, 0x5f, 0x37, 0xe0, 0xdc, 0x8e, 0xa3, 0x69, 0x5d, 0xee, 0x86, 0xd0,
0x1d, 0xac, 0x7b, 0x7d, 0xf2, 0x8a, 0x90, 0x3c, 0x77, 0x2f, 0x85, 0x07, 0xa7, 0x4a, 0x66, 0x5c,
0x25, 0xe7, 0x4f, 0x74, 0x95, 0x9c, 0xb8, 0xd9, 0x2c, 0x0c, 0x70, 0xb3, 0x99, 0x7a, 0x2d, 0x3c,
0x74, 0x82, 0x6b, 0xe1, 0x93, 0xdc, 0xe3, 0xa6, 0x24, 0xb1, 0x9e, 0xcf, 0x0a, 0x5f, 0x81, 0x39,
0x21, 0x46, 0xf9, 0x15, 0xab, 0x4e, 0x2d, 0x43, 0xd3, 0x88, 0xb5, 0xe4, 0x74, 0x3a, 0x5d, 0xf9,
0x5d, 0x18, 0x8f, 0x3e, 0x1e, 0x70, 0x57, 0xda, 0x7d, 0xbf, 0x20, 0x2e, 0xb1, 0x42, 0x2b, 0xed,
0x8e, 0x63, 0x9f, 0x43, 0xfe, 0x6b, 0x09, 0x66, 0xd2, 0x1f, 0x09, 0x22, 0x0d, 0xc6, 0x3b, 0xca,
0x41, 0xf8, 0xe1, 0xa6, 0x74, 0xc2, 0xe3, 0x05, 0x7e, 0x6b, 0xbc, 0x1a, 0xc1, 0xc2, 0x31, 0x6c,
0xf9, 0x07, 0x09, 0x66, 0x33, 0xee, 0x6b, 0x4f, 0xd7, 0x12, 0xf4, 0x21, 0x94, 0x3a, 0xca, 0x41,
0xc3, 0xb1, 0xda, 0xe4, 0xc4, 0x07, 0x2a, 0x3c, 0x63, 0xac, 0x0a, 0x14, 0xec, 0xe3, 0xc9, 0x5f,
0x48, 0x50, 0xce, 0x6a, 0x6d, 0xd1, 0xcd, 0xc8, 0xcd, 0xf2, 0xab, 0xb1, 0x9b, 0xe5, 0xa9, 0x84,
0xdc, 0x0b, 0xba, 0x57, 0xfe, 0x0f, 0x09, 0x66, 0xd2, 0x5b, 0x7c, 0xf4, 0x66, 0xc4, 0xc2, 0x4a,
0xcc, 0xc2, 0x89, 0x98, 0x94, 0xb0, 0xef, 0x23, 0x18, 0x17, 0x1f, 0x02, 0x02, 0x46, 0x78, 0x55,
0x4e, 0xcb, 0x95, 0x02, 0xc2, 0x6b, 0x7c, 0xf9, 0x7a, 0x45, 0xc7, 0x70, 0x0c, 0x4d, 0xfe, 0x9b,
0x1c, 0x0c, 0x35, 0x9a, 0x8a, 0x46, 0x4e, 0xa1, 0xcd, 0x7a, 0x3f, 0xd2, 0x66, 0xf5, 0xfa, 0x27,
0x0b, 0x6e, 0x55, 0x66, 0x87, 0x85, 0x63, 0x1d, 0xd6, 0xeb, 0x7d, 0xa1, 0x1d, 0xdf, 0x5c, 0xfd,
0x01, 0x8c, 0xf8, 0x4a, 0x07, 0xcb, 0xf9, 0xf2, 0xbf, 0xe6, 0x60, 0x34, 0xa4, 0x62, 0xc0, 0x8a,
0xb1, 0x13, 0xa9, 0xb4, 0xfd, 0xfc, 0x6b, 0x5b, 0x48, 0x57, 0xd5, 0xab, 0xad, 0xee, 0x23, 0xc1,
0xe0, 0x59, 0x58, 0xb2, 0xe4, 0xbe, 0x0b, 0xe3, 0x94, 0xff, 0xeb, 0x97, 0x7f, 0x0c, 0x99, 0xe7,
0xb1, 0xe8, 0x3f, 0x2d, 0xdd, 0x8c, 0x50, 0x71, 0x8c, 0x7b, 0xee, 0x0e, 0x8c, 0x45, 0x94, 0x0d,
0xf4, 0xc6, 0xef, 0xbf, 0x25, 0x78, 0xb5, 0xe7, 0x47, 0x22, 0xaa, 0x47, 0x36, 0x49, 0x35, 0xb6,
0x49, 0xe6, 0xb3, 0x01, 0x5e, 0xdc, 0x5b, 0x91, 0xfa, 0xb5, 0xa7, 0xdf, 0xcf, 0x9f, 0xf9, 0xf6,
0xfb, 0xf9, 0x33, 0xdf, 0x7d, 0x3f, 0x7f, 0xe6, 0x2f, 0x8f, 0xe6, 0xa5, 0xa7, 0x47, 0xf3, 0xd2,
0xb7, 0x47, 0xf3, 0xd2, 0x77, 0x47, 0xf3, 0xd2, 0xcf, 0x8e, 0xe6, 0xa5, 0x7f, 0xf8, 0x61, 0xfe,
0xcc, 0x87, 0x45, 0x01, 0xf7, 0xeb, 0x00, 0x00, 0x00, 0xff, 0xff, 0x22, 0xba, 0x16, 0x0c, 0x99,
0x3b, 0x00, 0x00,
0x4d, 0x73, 0x80, 0x20, 0xc8, 0x21, 0x08, 0x10, 0x20, 0x41, 0x92, 0x83, 0xf3, 0x71, 0x8b, 0x2f,
0x39, 0x25, 0x48, 0x6e, 0xc9, 0xc1, 0x30, 0x10, 0xc0, 0x0b, 0x08, 0x0b, 0x2f, 0xe0, 0xdb, 0xfa,
0x44, 0xac, 0xe9, 0xd3, 0x62, 0xff, 0x81, 0x85, 0x0e, 0x8b, 0x45, 0x55, 0x57, 0x7f, 0x77, 0x73,
0x66, 0x68, 0x89, 0x58, 0x2c, 0xf6, 0xc6, 0xa9, 0xf7, 0xde, 0xef, 0xbd, 0x7a, 0xf5, 0xea, 0xbd,
0xd7, 0x55, 0x45, 0xb8, 0xb7, 0x77, 0xdb, 0xae, 0xaa, 0x46, 0x6d, 0xcf, 0xd9, 0x26, 0x96, 0x4e,
0x28, 0xb1, 0x6b, 0xfb, 0x44, 0x6f, 0x19, 0x56, 0x4d, 0x10, 0x14, 0x53, 0xad, 0x91, 0x03, 0x4a,
0x74, 0x5b, 0x35, 0x74, 0xbb, 0xb6, 0x7f, 0x7d, 0x9b, 0x50, 0xe5, 0x7a, 0xad, 0x4d, 0x74, 0x62,
0x29, 0x94, 0xb4, 0xaa, 0xa6, 0x65, 0x50, 0x03, 0x5d, 0x70, 0xd9, 0xab, 0x8a, 0xa9, 0x56, 0x03,
0xf6, 0xaa, 0x60, 0x9f, 0xbb, 0xd6, 0x56, 0xe9, 0xae, 0xb3, 0x5d, 0x6d, 0x1a, 0x9d, 0x5a, 0xdb,
0x68, 0x1b, 0x35, 0x2e, 0xb5, 0xed, 0xec, 0xf0, 0x5f, 0xfc, 0x07, 0xff, 0xcb, 0x45, 0x9b, 0x93,
0x43, 0xca, 0x9b, 0x86, 0x45, 0x6a, 0xfb, 0x09, 0x8d, 0x73, 0x57, 0x42, 0x3c, 0xa6, 0xa1, 0xa9,
0xcd, 0x6e, 0x96, 0x71, 0x73, 0x6f, 0x05, 0xac, 0x1d, 0xa5, 0xb9, 0xab, 0xea, 0xc4, 0xea, 0xd6,
0xcc, 0xbd, 0x36, 0x97, 0xb5, 0x88, 0x6d, 0x38, 0x56, 0x93, 0x0c, 0x24, 0x65, 0xd7, 0x3a, 0x84,
0x2a, 0x69, 0x66, 0xd5, 0xb2, 0xa4, 0x2c, 0x47, 0xa7, 0x6a, 0x27, 0xa9, 0xe6, 0x56, 0x2f, 0x01,
0xbb, 0xb9, 0x4b, 0x3a, 0x4a, 0x42, 0xee, 0xcd, 0x2c, 0x39, 0x87, 0xaa, 0x5a, 0x4d, 0xd5, 0xa9,
0x4d, 0xad, 0xb8, 0x90, 0x7c, 0x07, 0xa6, 0x16, 0x34, 0xcd, 0xf8, 0x94, 0xb4, 0xee, 0x69, 0xe4,
0xe0, 0x91, 0xa1, 0x39, 0x1d, 0x82, 0x2e, 0xc1, 0x70, 0xcb, 0x52, 0xf7, 0x89, 0x55, 0x96, 0x2e,
0x4a, 0x97, 0x47, 0xea, 0xe3, 0x4f, 0x0f, 0x2b, 0x67, 0x8e, 0x0e, 0x2b, 0xc3, 0x4b, 0x7c, 0x14,
0x0b, 0xaa, 0x7c, 0x17, 0x26, 0x84, 0xf0, 0x03, 0xc3, 0xa6, 0x1b, 0x0a, 0xdd, 0x45, 0x37, 0x00,
0x4c, 0x85, 0xee, 0x6e, 0x58, 0x64, 0x47, 0x3d, 0x10, 0xe2, 0x48, 0x88, 0xc3, 0x86, 0x4f, 0xc1,
0x21, 0x2e, 0xf9, 0xdf, 0x24, 0x78, 0x79, 0xd1, 0xb1, 0xa9, 0xd1, 0x59, 0x25, 0xd4, 0x52, 0x9b,
0x8b, 0x8e, 0x65, 0x11, 0x9d, 0x36, 0xa8, 0x42, 0x1d, 0x1b, 0x5d, 0x84, 0x82, 0xae, 0x74, 0x88,
0xc0, 0x3a, 0x2b, 0xb0, 0x0a, 0x6b, 0x4a, 0x87, 0x60, 0x4e, 0x41, 0x1f, 0xc2, 0xd0, 0xbe, 0xa2,
0x39, 0xa4, 0x9c, 0xbb, 0x28, 0x5d, 0x1e, 0xbd, 0x51, 0xad, 0x06, 0xa1, 0xe7, 0x3b, 0xa2, 0x6a,
0xee, 0xb5, 0x79, 0x2c, 0x7a, 0xab, 0x5b, 0x7d, 0xe8, 0x28, 0x3a, 0x55, 0x69, 0xb7, 0x7e, 0x4e,
0x40, 0x9e, 0x15, 0x7a, 0x1f, 0x31, 0x2c, 0xec, 0x42, 0xca, 0x7f, 0x09, 0x17, 0x32, 0x4d, 0x5b,
0x51, 0x6d, 0x8a, 0x1e, 0xc3, 0x90, 0x4a, 0x49, 0xc7, 0x2e, 0x4b, 0x17, 0xf3, 0x97, 0x47, 0x6f,
0xdc, 0xae, 0x1e, 0x1b, 0xf7, 0xd5, 0x4c, 0xb0, 0xfa, 0x98, 0x30, 0x63, 0x68, 0x99, 0xc1, 0x61,
0x17, 0x55, 0xfe, 0x27, 0x09, 0x50, 0x58, 0x66, 0x53, 0xb1, 0xda, 0x84, 0xf6, 0xe1, 0x94, 0x3f,
0xfd, 0x61, 0x4e, 0x99, 0x16, 0x90, 0xa3, 0xae, 0xc2, 0x88, 0x4f, 0x4c, 0x98, 0x49, 0x9a, 0xc4,
0x9d, 0xf1, 0x28, 0xea, 0x8c, 0xeb, 0x03, 0x38, 0xc3, 0x45, 0xc9, 0xf0, 0xc2, 0x67, 0x39, 0x18,
0x59, 0x52, 0x48, 0xc7, 0xd0, 0x1b, 0x84, 0xa2, 0x8f, 0xa1, 0xc4, 0x36, 0x5b, 0x4b, 0xa1, 0x0a,
0x77, 0xc0, 0xe8, 0x8d, 0x37, 0x8e, 0x9b, 0x9d, 0x5d, 0x65, 0xdc, 0xd5, 0xfd, 0xeb, 0xd5, 0xf5,
0xed, 0x27, 0xa4, 0x49, 0x57, 0x09, 0x55, 0x82, 0x98, 0x0c, 0xc6, 0xb0, 0x8f, 0x8a, 0xd6, 0xa0,
0x60, 0x9b, 0xa4, 0x29, 0x7c, 0x77, 0xb5, 0xc7, 0x34, 0x7c, 0xcb, 0x1a, 0x26, 0x69, 0x06, 0x8b,
0xc1, 0x7e, 0x61, 0x8e, 0x83, 0x1e, 0xc1, 0xb0, 0xcd, 0x57, 0xb9, 0x9c, 0x4f, 0xac, 0xc6, 0xf1,
0x88, 0x6e, 0x6c, 0xf8, 0x1b, 0xd0, 0xfd, 0x8d, 0x05, 0x9a, 0xfc, 0xf3, 0x1c, 0x20, 0x9f, 0x77,
0xd1, 0xd0, 0x5b, 0x2a, 0x55, 0x0d, 0x1d, 0xbd, 0x03, 0x05, 0xda, 0x35, 0xbd, 0xe8, 0xb8, 0xe4,
0x19, 0xb4, 0xd9, 0x35, 0xc9, 0xb3, 0xc3, 0xca, 0x4c, 0x52, 0x82, 0x51, 0x30, 0x97, 0x41, 0x2b,
0xbe, 0xa9, 0x39, 0x2e, 0xfd, 0x56, 0x54, 0xf5, 0xb3, 0xc3, 0x4a, 0x4a, 0x2e, 0xae, 0xfa, 0x48,
0x51, 0x03, 0xd1, 0x3e, 0x20, 0x4d, 0xb1, 0xe9, 0xa6, 0xa5, 0xe8, 0xb6, 0xab, 0x49, 0xed, 0x10,
0xe1, 0x84, 0xd7, 0xfb, 0x5b, 0x34, 0x26, 0x51, 0x9f, 0x13, 0x56, 0xa0, 0x95, 0x04, 0x1a, 0x4e,
0xd1, 0xc0, 0x32, 0x98, 0x45, 0x14, 0xdb, 0xd0, 0xcb, 0x85, 0x68, 0x06, 0xc3, 0x7c, 0x14, 0x0b,
0x2a, 0xba, 0x02, 0xc5, 0x0e, 0xb1, 0x6d, 0xa5, 0x4d, 0xca, 0x43, 0x9c, 0x71, 0x42, 0x30, 0x16,
0x57, 0xdd, 0x61, 0xec, 0xd1, 0xe5, 0x2f, 0x24, 0x18, 0xf3, 0x3d, 0xc7, 0xa3, 0xfd, 0xcf, 0x12,
0x71, 0x58, 0xed, 0x6f, 0x4a, 0x4c, 0x9a, 0x47, 0xe1, 0xa4, 0xd0, 0x56, 0xf2, 0x46, 0x42, 0x31,
0xb8, 0xea, 0xed, 0xa5, 0x1c, 0xdf, 0x4b, 0x97, 0xfb, 0x0d, 0x99, 0x8c, 0x2d, 0xf4, 0xcf, 0x85,
0x90, 0xf9, 0x2c, 0x34, 0xd1, 0x63, 0x28, 0xd9, 0x44, 0x23, 0x4d, 0x6a, 0x58, 0xc2, 0xfc, 0x37,
0xfb, 0x34, 0x5f, 0xd9, 0x26, 0x5a, 0x43, 0x88, 0xd6, 0xcf, 0x32, 0xfb, 0xbd, 0x5f, 0xd8, 0x87,
0x44, 0x0f, 0xa1, 0x44, 0x49, 0xc7, 0xd4, 0x14, 0xea, 0xe5, 0xa0, 0xd7, 0xc2, 0x53, 0x60, 0x91,
0xc3, 0xc0, 0x36, 0x8c, 0xd6, 0xa6, 0x60, 0xe3, 0xdb, 0xc7, 0x77, 0x89, 0x37, 0x8a, 0x7d, 0x18,
0xb4, 0x0f, 0xe3, 0x8e, 0xd9, 0x62, 0x9c, 0x94, 0xd5, 0xb0, 0x76, 0x57, 0x44, 0xd2, 0xad, 0x7e,
0x7d, 0xb3, 0x15, 0x91, 0xae, 0xcf, 0x08, 0x5d, 0xe3, 0xd1, 0x71, 0x1c, 0xd3, 0x82, 0x16, 0x60,
0xa2, 0xa3, 0xea, 0x98, 0x28, 0xad, 0x6e, 0x83, 0x34, 0x0d, 0xbd, 0x65, 0xf3, 0xb0, 0x1a, 0xaa,
0xcf, 0x0a, 0x80, 0x89, 0xd5, 0x28, 0x19, 0xc7, 0xf9, 0xd1, 0xfb, 0x80, 0xbc, 0x69, 0xdc, 0x77,
0x4b, 0xb0, 0x6a, 0xe8, 0x3c, 0xe6, 0xf2, 0x41, 0x70, 0x6f, 0x26, 0x38, 0x70, 0x8a, 0x14, 0x5a,
0x81, 0x73, 0x16, 0xd9, 0x57, 0xd9, 0x1c, 0x1f, 0xa8, 0x36, 0x35, 0xac, 0xee, 0x8a, 0xda, 0x51,
0x69, 0x79, 0x98, 0xdb, 0x54, 0x3e, 0x3a, 0xac, 0x9c, 0xc3, 0x29, 0x74, 0x9c, 0x2a, 0x25, 0xff,
0xcb, 0x30, 0x4c, 0xc4, 0xf2, 0x0d, 0x7a, 0x04, 0x33, 0x4d, 0xb7, 0x38, 0xad, 0x39, 0x9d, 0x6d,
0x62, 0x35, 0x9a, 0xbb, 0xa4, 0xe5, 0x68, 0xa4, 0xc5, 0x03, 0x65, 0xa8, 0x3e, 0x2f, 0x2c, 0x9e,
0x59, 0x4c, 0xe5, 0xc2, 0x19, 0xd2, 0xcc, 0x0b, 0x3a, 0x1f, 0x5a, 0x55, 0x6d, 0xdb, 0xc7, 0xcc,
0x71, 0x4c, 0xdf, 0x0b, 0x6b, 0x09, 0x0e, 0x9c, 0x22, 0xc5, 0x6c, 0x6c, 0x11, 0x5b, 0xb5, 0x48,
0x2b, 0x6e, 0x63, 0x3e, 0x6a, 0xe3, 0x52, 0x2a, 0x17, 0xce, 0x90, 0x46, 0x37, 0x61, 0xd4, 0xd5,
0xc6, 0xd7, 0x4f, 0x2c, 0xb4, 0x5f, 0x0e, 0xd7, 0x02, 0x12, 0x0e, 0xf3, 0xb1, 0xa9, 0x19, 0xdb,
0x36, 0xb1, 0xf6, 0x49, 0x2b, 0x7b, 0x81, 0xd7, 0x13, 0x1c, 0x38, 0x45, 0x8a, 0x4d, 0xcd, 0x8d,
0xc0, 0xc4, 0xd4, 0x86, 0xa3, 0x53, 0xdb, 0x4a, 0xe5, 0xc2, 0x19, 0xd2, 0x2c, 0x8e, 0x5d, 0x93,
0x17, 0xf6, 0x15, 0x55, 0x53, 0xb6, 0x35, 0x52, 0x2e, 0x46, 0xe3, 0x78, 0x2d, 0x4a, 0xc6, 0x71,
0x7e, 0x74, 0x1f, 0xa6, 0xdc, 0xa1, 0x2d, 0x5d, 0xf1, 0x41, 0x4a, 0x1c, 0xe4, 0x65, 0x01, 0x32,
0xb5, 0x16, 0x67, 0xc0, 0x49, 0x19, 0xf4, 0x0e, 0x8c, 0x37, 0x0d, 0x4d, 0xe3, 0xf1, 0xb8, 0x68,
0x38, 0x3a, 0x2d, 0x8f, 0x70, 0x14, 0xc4, 0xf6, 0xe3, 0x62, 0x84, 0x82, 0x63, 0x9c, 0x88, 0x00,
0x34, 0xbd, 0x82, 0x63, 0x97, 0xa1, 0xaf, 0x5e, 0x23, 0x59, 0xf4, 0x82, 0x1e, 0xc0, 0x1f, 0xb2,
0x71, 0x08, 0x58, 0xfe, 0xb1, 0x04, 0xb3, 0x19, 0xa9, 0x03, 0xbd, 0x17, 0x29, 0xb1, 0xbf, 0x1f,
0x2b, 0xb1, 0xe7, 0x33, 0xc4, 0x42, 0x75, 0x56, 0x87, 0x31, 0x8b, 0xcd, 0x4a, 0x6f, 0xbb, 0x2c,
0x22, 0x47, 0xde, 0xec, 0x31, 0x0d, 0x1c, 0x96, 0x09, 0x72, 0xfe, 0xd4, 0xd1, 0x61, 0x65, 0x2c,
0x42, 0xc3, 0x51, 0x78, 0xf9, 0x5f, 0x73, 0x00, 0x4b, 0xc4, 0xd4, 0x8c, 0x6e, 0x87, 0xe8, 0xa7,
0xd1, 0x43, 0xad, 0x47, 0x7a, 0xa8, 0x6b, 0xbd, 0x96, 0xc7, 0x37, 0x2d, 0xb3, 0x89, 0xfa, 0x93,
0x58, 0x13, 0x55, 0xeb, 0x1f, 0xf2, 0xf8, 0x2e, 0xea, 0xa7, 0x79, 0x98, 0x0e, 0x98, 0x83, 0x36,
0xea, 0x4e, 0x64, 0x8d, 0x7f, 0x2f, 0xb6, 0xc6, 0xb3, 0x29, 0x22, 0x2f, 0xac, 0x8f, 0x7a, 0xfe,
0xfd, 0x0c, 0x7a, 0x02, 0xe3, 0xac, 0x71, 0x72, 0xc3, 0x83, 0xb7, 0x65, 0xc3, 0x03, 0xb7, 0x65,
0x7e, 0x01, 0x5d, 0x89, 0x20, 0xe1, 0x18, 0x72, 0x46, 0x1b, 0x58, 0x7c, 0xd1, 0x6d, 0xa0, 0xfc,
0xa5, 0x04, 0xe3, 0xc1, 0x32, 0x9d, 0x42, 0xd3, 0xb6, 0x16, 0x6d, 0xda, 0xae, 0xf4, 0x1d, 0xa2,
0x19, 0x5d, 0xdb, 0x2f, 0x59, 0x83, 0xef, 0x33, 0xb1, 0x0d, 0xbe, 0xad, 0x34, 0xf7, 0xfa, 0xf8,
0xfc, 0xfb, 0x4c, 0x02, 0x24, 0xaa, 0xc0, 0x82, 0xae, 0x1b, 0x54, 0x71, 0x73, 0xa5, 0x6b, 0xd6,
0x72, 0xdf, 0x66, 0x79, 0x1a, 0xab, 0x5b, 0x09, 0xac, 0xbb, 0x3a, 0xb5, 0xba, 0xc1, 0x8a, 0x24,
0x19, 0x70, 0x8a, 0x01, 0x48, 0x01, 0xb0, 0x04, 0xe6, 0xa6, 0x21, 0x36, 0xf2, 0xb5, 0x3e, 0x72,
0x1e, 0x13, 0x58, 0x34, 0xf4, 0x1d, 0xb5, 0x1d, 0xa4, 0x1d, 0xec, 0x03, 0xe1, 0x10, 0xe8, 0xdc,
0x5d, 0x98, 0xcd, 0xb0, 0x16, 0x4d, 0x42, 0x7e, 0x8f, 0x74, 0x5d, 0xb7, 0x61, 0xf6, 0x27, 0x3a,
0x17, 0xfe, 0x4c, 0x1e, 0x11, 0x5f, 0xb8, 0xef, 0xe4, 0x6e, 0x4b, 0xf2, 0x17, 0x43, 0xe1, 0xd8,
0xe1, 0x1d, 0xf3, 0x65, 0x28, 0x59, 0xc4, 0xd4, 0xd4, 0xa6, 0x62, 0x8b, 0x46, 0x88, 0x37, 0xbf,
0x58, 0x8c, 0x61, 0x9f, 0x1a, 0xe9, 0xad, 0x73, 0x2f, 0xb6, 0xb7, 0xce, 0x3f, 0x9f, 0xde, 0xfa,
0xcf, 0xa1, 0x64, 0x7b, 0x5d, 0x75, 0x81, 0x43, 0x5e, 0x1f, 0x20, 0xbf, 0x8a, 0x86, 0xda, 0x57,
0xe0, 0xb7, 0xd2, 0x3e, 0x68, 0x5a, 0x13, 0x3d, 0x34, 0x60, 0x13, 0xfd, 0x5c, 0x1b, 0x5f, 0x96,
0x53, 0x4d, 0xc5, 0xb1, 0x49, 0x8b, 0x27, 0xa2, 0x52, 0x90, 0x53, 0x37, 0xf8, 0x28, 0x16, 0x54,
0xf4, 0x38, 0x12, 0xb2, 0xa5, 0x93, 0x84, 0xec, 0x78, 0x76, 0xb8, 0xa2, 0x2d, 0x98, 0x35, 0x2d,
0xa3, 0x6d, 0x11, 0xdb, 0x5e, 0x22, 0x4a, 0x4b, 0x53, 0x75, 0xe2, 0xf9, 0xc7, 0xed, 0x88, 0xce,
0x1f, 0x1d, 0x56, 0x66, 0x37, 0xd2, 0x59, 0x70, 0x96, 0xac, 0xfc, 0xb4, 0x00, 0x93, 0xf1, 0x0a,
0x98, 0xd1, 0xa4, 0x4a, 0x27, 0x6a, 0x52, 0xaf, 0x86, 0x36, 0x83, 0xdb, 0xc1, 0xfb, 0xab, 0x9f,
0xb2, 0x21, 0x16, 0x60, 0x42, 0x64, 0x03, 0x8f, 0x28, 0xda, 0x74, 0x7f, 0xf5, 0xb7, 0xa2, 0x64,
0x1c, 0xe7, 0x67, 0xad, 0x67, 0xd0, 0x51, 0x7a, 0x20, 0x85, 0x68, 0xeb, 0xb9, 0x10, 0x67, 0xc0,
0x49, 0x19, 0xb4, 0x0a, 0xd3, 0x8e, 0x9e, 0x84, 0x72, 0xa3, 0xf1, 0xbc, 0x80, 0x9a, 0xde, 0x4a,
0xb2, 0xe0, 0x34, 0x39, 0xb4, 0x13, 0xe9, 0x46, 0x87, 0x79, 0x86, 0xbd, 0xd1, 0xf7, 0xde, 0xe9,
0xbb, 0x1d, 0x45, 0x77, 0x60, 0xcc, 0xe2, 0xdf, 0x1d, 0x9e, 0xc1, 0x6e, 0xef, 0xfe, 0x92, 0x10,
0x1b, 0xc3, 0x61, 0x22, 0x8e, 0xf2, 0xa6, 0xb4, 0xdb, 0xa5, 0x7e, 0xdb, 0x6d, 0xf9, 0xff, 0xa5,
0x70, 0x11, 0xf2, 0x5b, 0xe0, 0x5e, 0xa7, 0x4c, 0x09, 0x89, 0x50, 0x77, 0x64, 0xa4, 0x77, 0xbf,
0xb7, 0x06, 0xea, 0x7e, 0x83, 0xe2, 0xd9, 0xbb, 0xfd, 0xfd, 0x5c, 0x82, 0x99, 0x7b, 0x8d, 0xfb,
0x96, 0xe1, 0x98, 0x9e, 0x39, 0xeb, 0xa6, 0xeb, 0xd7, 0xb7, 0xa1, 0x60, 0x39, 0x9a, 0x37, 0x8f,
0xd7, 0xbc, 0x79, 0x60, 0x47, 0x63, 0xf3, 0x98, 0x8e, 0x49, 0xb9, 0x93, 0x60, 0x02, 0x68, 0x0d,
0x86, 0x2d, 0x45, 0x6f, 0x13, 0xaf, 0xac, 0x5e, 0xea, 0x61, 0xfd, 0xf2, 0x12, 0x66, 0xec, 0xa1,
0xe6, 0x8d, 0x4b, 0x63, 0x81, 0x22, 0xff, 0xbd, 0x04, 0x13, 0x0f, 0x36, 0x37, 0x37, 0x96, 0x75,
0xbe, 0xa3, 0xf9, 0x79, 0xfa, 0x45, 0x28, 0x98, 0x0a, 0xdd, 0x8d, 0x57, 0x7a, 0x46, 0xc3, 0x9c,
0x82, 0x3e, 0x80, 0x22, 0xcb, 0x24, 0x44, 0x6f, 0xf5, 0xd9, 0x6a, 0x0b, 0xf8, 0xba, 0x2b, 0x14,
0x74, 0x88, 0x62, 0x00, 0x7b, 0x70, 0xf2, 0x1e, 0x9c, 0x0b, 0x99, 0xc3, 0xfc, 0xc1, 0x8f, 0x81,
0x51, 0x03, 0x86, 0x98, 0x66, 0xef, 0x94, 0xb7, 0xd7, 0x61, 0x66, 0x6c, 0x4a, 0x41, 0xa7, 0xc3,
0x7e, 0xd9, 0xd8, 0xc5, 0x92, 0x57, 0x61, 0x8c, 0x5f, 0x22, 0x18, 0x16, 0xe5, 0x6e, 0x41, 0x17,
0x20, 0xdf, 0x51, 0x75, 0x51, 0x67, 0x47, 0x85, 0x4c, 0x9e, 0xd5, 0x08, 0x36, 0xce, 0xc9, 0xca,
0x81, 0xc8, 0x3c, 0x01, 0x59, 0x39, 0xc0, 0x6c, 0x5c, 0xbe, 0x0f, 0x45, 0xe1, 0xee, 0x30, 0x50,
0xfe, 0x78, 0xa0, 0x7c, 0x0a, 0xd0, 0x3a, 0x14, 0x97, 0x37, 0xea, 0x9a, 0xe1, 0x76, 0x5d, 0x4d,
0xb5, 0x65, 0xc5, 0xd7, 0x62, 0x71, 0x79, 0x09, 0x63, 0x4e, 0x41, 0x32, 0x0c, 0x93, 0x83, 0x26,
0x31, 0x29, 0x8f, 0x88, 0x91, 0x3a, 0xb0, 0x55, 0xbe, 0xcb, 0x47, 0xb0, 0xa0, 0xc8, 0xff, 0x90,
0x83, 0xa2, 0x70, 0xc7, 0x29, 0x7c, 0x85, 0xad, 0x44, 0xbe, 0xc2, 0x5e, 0xef, 0x2f, 0x34, 0x32,
0x3f, 0xc1, 0x36, 0x63, 0x9f, 0x60, 0x57, 0xfb, 0xc4, 0x3b, 0xfe, 0xfb, 0xeb, 0x7f, 0x24, 0x18,
0x8f, 0x06, 0x25, 0xba, 0x09, 0xa3, 0xac, 0xe0, 0xa8, 0x4d, 0xb2, 0x16, 0xf4, 0xb9, 0xfe, 0x21,
0x4c, 0x23, 0x20, 0xe1, 0x30, 0x1f, 0x6a, 0xfb, 0x62, 0x2c, 0x8e, 0xc4, 0xa4, 0xb3, 0x5d, 0xea,
0x50, 0x55, 0xab, 0xba, 0x17, 0x63, 0xd5, 0x65, 0x9d, 0xae, 0x5b, 0x0d, 0x6a, 0xa9, 0x7a, 0x3b,
0xa1, 0x88, 0x07, 0x65, 0x18, 0x59, 0xfe, 0x3f, 0x09, 0x46, 0x85, 0xc9, 0xa7, 0xf0, 0x55, 0xf1,
0xc7, 0xd1, 0xaf, 0x8a, 0x4b, 0x7d, 0x6e, 0xf0, 0xf4, 0x4f, 0x8a, 0xff, 0x08, 0x4c, 0x67, 0x5b,
0x9a, 0x45, 0xf5, 0xae, 0x61, 0xd3, 0x78, 0x54, 0xb3, 0xcd, 0x88, 0x39, 0x05, 0x39, 0x30, 0xa9,
0xc6, 0x72, 0x80, 0x70, 0x6d, 0xad, 0x3f, 0x4b, 0x7c, 0xb1, 0x7a, 0x59, 0xc0, 0x4f, 0xc6, 0x29,
0x38, 0xa1, 0x42, 0x26, 0x90, 0xe0, 0x42, 0x0f, 0xa1, 0xb0, 0x4b, 0xa9, 0x99, 0x72, 0x5e, 0xdd,
0x23, 0xf3, 0x04, 0x26, 0x94, 0xf8, 0xec, 0x36, 0x37, 0x37, 0x30, 0x87, 0x92, 0x7f, 0x15, 0xf8,
0xa3, 0xe1, 0xc6, 0xb8, 0x9f, 0x4f, 0xa5, 0x93, 0xe4, 0xd3, 0xd1, 0xb4, 0x5c, 0x8a, 0x1e, 0x40,
0x9e, 0x6a, 0xfd, 0x7e, 0x16, 0x0a, 0xc4, 0xcd, 0x95, 0x46, 0x90, 0x90, 0x36, 0x57, 0x1a, 0x98,
0x41, 0xa0, 0x75, 0x18, 0x62, 0xd5, 0x87, 0x6d, 0xc1, 0x7c, 0xff, 0x5b, 0x9a, 0xcd, 0x3f, 0x08,
0x08, 0xf6, 0xcb, 0xc6, 0x2e, 0x8e, 0xfc, 0x09, 0x8c, 0x45, 0xf6, 0x29, 0xfa, 0x18, 0xce, 0x6a,
0x86, 0xd2, 0xaa, 0x2b, 0x9a, 0xa2, 0x37, 0x89, 0x77, 0x39, 0x70, 0x29, 0xed, 0x0b, 0x63, 0x25,
0xc4, 0x27, 0x76, 0xb9, 0x7f, 0x9d, 0x1a, 0xa6, 0xe1, 0x08, 0xa2, 0xac, 0x00, 0x04, 0x73, 0x44,
0x15, 0x18, 0x62, 0x71, 0xe6, 0xd6, 0x93, 0x91, 0xfa, 0x08, 0xb3, 0x90, 0x85, 0x9f, 0x8d, 0xdd,
0x71, 0x74, 0x03, 0xc0, 0x26, 0x4d, 0x8b, 0x50, 0x9e, 0x0c, 0x72, 0xd1, 0x4b, 0xe5, 0x86, 0x4f,
0xc1, 0x21, 0x2e, 0xf9, 0x47, 0x12, 0x8c, 0xad, 0x11, 0xfa, 0xa9, 0x61, 0xed, 0x6d, 0xf0, 0xc7,
0x00, 0xa7, 0x90, 0x6c, 0x71, 0x24, 0xd9, 0xbe, 0xd1, 0x63, 0x65, 0x22, 0xd6, 0x65, 0xa5, 0x5c,
0xf9, 0x4b, 0x09, 0x66, 0x23, 0x9c, 0x77, 0x83, 0xad, 0xbb, 0x05, 0x43, 0xa6, 0x61, 0x51, 0xaf,
0x10, 0x0f, 0xa4, 0x90, 0xa5, 0xb1, 0x50, 0x29, 0x66, 0x30, 0xd8, 0x45, 0x43, 0x2b, 0x90, 0xa3,
0x86, 0x08, 0xd5, 0xc1, 0x30, 0x09, 0xb1, 0xea, 0x20, 0x30, 0x73, 0x9b, 0x06, 0xce, 0x51, 0x83,
0x2d, 0x44, 0x39, 0xc2, 0x15, 0x4e, 0x3e, 0x2f, 0x68, 0x06, 0x18, 0x0a, 0x3b, 0x96, 0xd1, 0x39,
0xf1, 0x1c, 0xfc, 0x85, 0xb8, 0x67, 0x19, 0x1d, 0xcc, 0xb1, 0xe4, 0xaf, 0x24, 0x98, 0x8a, 0x70,
0x9e, 0x42, 0xe2, 0x7f, 0x18, 0x4d, 0xfc, 0x57, 0x07, 0x99, 0x48, 0x46, 0xfa, 0xff, 0x2a, 0x17,
0x9b, 0x06, 0x9b, 0x30, 0xda, 0x81, 0x51, 0xd3, 0x68, 0x35, 0x9e, 0xc3, 0x75, 0xe0, 0x04, 0xab,
0x9b, 0x1b, 0x01, 0x16, 0x0e, 0x03, 0xa3, 0x03, 0x98, 0xd2, 0x95, 0x0e, 0xb1, 0x4d, 0xa5, 0x49,
0x1a, 0xcf, 0xe1, 0x80, 0xe4, 0x25, 0x7e, 0xdf, 0x10, 0x47, 0xc4, 0x49, 0x25, 0x68, 0x15, 0x8a,
0xaa, 0xc9, 0xfb, 0x38, 0xd1, 0xbb, 0xf4, 0xac, 0xa2, 0x6e, 0xd7, 0xe7, 0xe6, 0x73, 0xf1, 0x03,
0x7b, 0x18, 0xf2, 0x7f, 0xc6, 0xa3, 0x81, 0xc5, 0x1f, 0xba, 0x0f, 0x25, 0xfe, 0xac, 0xa6, 0x69,
0x68, 0xde, 0xcd, 0x00, 0x5b, 0xd9, 0x0d, 0x31, 0xf6, 0xec, 0xb0, 0x72, 0x3e, 0xe5, 0xd0, 0xd7,
0x23, 0x63, 0x5f, 0x18, 0xad, 0x41, 0xc1, 0xfc, 0x21, 0x1d, 0x0c, 0x2f, 0x72, 0xbc, 0x6d, 0xe1,
0x38, 0xf2, 0x5f, 0xe7, 0x63, 0xe6, 0xf2, 0x52, 0xf7, 0xe4, 0xb9, 0xad, 0xba, 0xdf, 0x31, 0x65,
0xae, 0xfc, 0x36, 0x14, 0x45, 0x85, 0x17, 0xc1, 0xfc, 0xf6, 0x20, 0xc1, 0x1c, 0xae, 0x62, 0xfe,
0x07, 0x8b, 0x37, 0xe8, 0x01, 0xa3, 0x8f, 0x60, 0x98, 0xb8, 0x2a, 0xdc, 0xda, 0x78, 0x6b, 0x10,
0x15, 0x41, 0x5e, 0x0d, 0x1a, 0x55, 0x31, 0x26, 0x50, 0xd1, 0x7b, 0xcc, 0x5f, 0x8c, 0x97, 0x7d,
0x04, 0xda, 0xe5, 0x02, 0x2f, 0x57, 0x17, 0xdc, 0x69, 0xfb, 0xc3, 0xcf, 0x0e, 0x2b, 0x10, 0xfc,
0xc4, 0x61, 0x09, 0xf9, 0x27, 0x12, 0x4c, 0x71, 0x0f, 0x35, 0x1d, 0x4b, 0xa5, 0xdd, 0x53, 0x2b,
0x4c, 0x8f, 0x22, 0x85, 0xe9, 0xad, 0x1e, 0x6e, 0x49, 0x58, 0x98, 0x59, 0x9c, 0xbe, 0x96, 0xe0,
0xa5, 0x04, 0xf7, 0x29, 0xe4, 0xc5, 0xad, 0x68, 0x5e, 0x7c, 0x63, 0xd0, 0x09, 0x65, 0xbd, 0x91,
0x18, 0x4b, 0x99, 0x0e, 0xdf, 0x29, 0x37, 0x00, 0x4c, 0x4b, 0xdd, 0x57, 0x35, 0xd2, 0x16, 0x97,
0xe0, 0xa5, 0xd0, 0xb3, 0x36, 0x9f, 0x82, 0x43, 0x5c, 0xc8, 0x86, 0x99, 0x16, 0xd9, 0x51, 0x1c,
0x8d, 0x2e, 0xb4, 0x5a, 0x8b, 0x8a, 0xa9, 0x6c, 0xab, 0x9a, 0x4a, 0x55, 0x71, 0x5c, 0x30, 0x52,
0xbf, 0xe3, 0x5e, 0x4e, 0xa7, 0x71, 0x3c, 0x3b, 0xac, 0x5c, 0x48, 0xbb, 0x1d, 0xf2, 0x58, 0xba,
0x38, 0x03, 0x1a, 0x75, 0xa1, 0x6c, 0x91, 0x4f, 0x1c, 0xd5, 0x22, 0xad, 0x25, 0xcb, 0x30, 0x23,
0x6a, 0xf3, 0x5c, 0xed, 0x1f, 0x1e, 0x1d, 0x56, 0xca, 0x38, 0x83, 0xa7, 0xb7, 0xe2, 0x4c, 0x78,
0xf4, 0x04, 0xa6, 0x15, 0xf7, 0x35, 0x60, 0x44, 0xab, 0xbb, 0x4b, 0x6e, 0x1f, 0x1d, 0x56, 0xa6,
0x17, 0x92, 0xe4, 0xde, 0x0a, 0xd3, 0x40, 0x51, 0x0d, 0x8a, 0xfb, 0xfc, 0xad, 0xa2, 0x5d, 0x1e,
0xe2, 0xf8, 0xac, 0x10, 0x14, 0xdd, 0xe7, 0x8b, 0x0c, 0x73, 0xf8, 0x5e, 0x83, 0xef, 0x3e, 0x8f,
0x8b, 0x7d, 0x50, 0xb2, 0x5e, 0x52, 0xec, 0x78, 0x7e, 0x62, 0x5c, 0x0a, 0xb2, 0xd6, 0x83, 0x80,
0x84, 0xc3, 0x7c, 0xe8, 0x31, 0x8c, 0xec, 0x8a, 0x53, 0x09, 0xbb, 0x5c, 0xec, 0xab, 0x08, 0x47,
0x4e, 0x31, 0xea, 0x53, 0x42, 0xc5, 0x88, 0x37, 0x6c, 0xe3, 0x00, 0x11, 0x5d, 0x81, 0x22, 0xff,
0xb1, 0xbc, 0xc4, 0x8f, 0xe3, 0x4a, 0x41, 0x6e, 0x7b, 0xe0, 0x0e, 0x63, 0x8f, 0xee, 0xb1, 0x2e,
0x6f, 0x2c, 0xf2, 0x63, 0xe1, 0x18, 0xeb, 0xf2, 0xc6, 0x22, 0xf6, 0xe8, 0xe8, 0x63, 0x28, 0xda,
0x64, 0x45, 0xd5, 0x9d, 0x83, 0x32, 0xf4, 0x75, 0xa9, 0xdc, 0xb8, 0xcb, 0xb9, 0x63, 0x07, 0x63,
0x81, 0x06, 0x41, 0xc7, 0x1e, 0x2c, 0xda, 0x85, 0x11, 0xcb, 0xd1, 0x17, 0xec, 0x2d, 0x9b, 0x58,
0xe5, 0x51, 0xae, 0xa3, 0x57, 0x3a, 0xc7, 0x1e, 0x7f, 0x5c, 0x8b, 0xef, 0x21, 0x9f, 0x03, 0x07,
0xe0, 0xe8, 0xef, 0x24, 0x40, 0xb6, 0x63, 0x9a, 0x1a, 0xe9, 0x10, 0x9d, 0x2a, 0x1a, 0x3f, 0x8b,
0xb3, 0xcb, 0x67, 0xb9, 0xce, 0x3f, 0xea, 0x35, 0xaf, 0x84, 0x60, 0x5c, 0xb9, 0x7f, 0xe8, 0x9d,
0x64, 0xc5, 0x29, 0x7a, 0x99, 0x6b, 0x77, 0x6c, 0xfe, 0x77, 0x79, 0xac, 0x2f, 0xd7, 0xa6, 0x9f,
0x39, 0x06, 0xae, 0x15, 0x74, 0xec, 0xc1, 0xa2, 0x47, 0x30, 0x63, 0x11, 0xa5, 0xb5, 0xae, 0x6b,
0x5d, 0x6c, 0x18, 0xf4, 0x9e, 0xaa, 0x11, 0xbb, 0x6b, 0x53, 0xd2, 0x29, 0x8f, 0xf3, 0x65, 0xf7,
0xdf, 0x7e, 0xe0, 0x54, 0x2e, 0x9c, 0x21, 0x8d, 0x3a, 0x50, 0xf1, 0x52, 0x06, 0xdb, 0x4f, 0x7e,
0xce, 0xba, 0x6b, 0x37, 0x15, 0xcd, 0xbd, 0x07, 0x98, 0xe0, 0x0a, 0x5e, 0x3b, 0x3a, 0xac, 0x54,
0x96, 0x8e, 0x67, 0xc5, 0xbd, 0xb0, 0xd0, 0x07, 0x50, 0x56, 0xb2, 0xf4, 0x4c, 0x72, 0x3d, 0xaf,
0xb0, 0x3c, 0x94, 0xa9, 0x20, 0x53, 0x1a, 0x51, 0x98, 0x54, 0xa2, 0x8f, 0x8e, 0xed, 0xf2, 0x54,
0x5f, 0x07, 0x91, 0xb1, 0xb7, 0xca, 0xc1, 0x61, 0x44, 0x8c, 0x60, 0xe3, 0x84, 0x06, 0xf4, 0x17,
0x80, 0x94, 0xf8, 0x3b, 0x69, 0xbb, 0x8c, 0xfa, 0x2a, 0x3f, 0x89, 0x07, 0xd6, 0x41, 0xd8, 0x25,
0x48, 0x36, 0x4e, 0xd1, 0xc3, 0x1f, 0x6f, 0x88, 0xa3, 0xfc, 0xd3, 0x79, 0x00, 0x3b, 0xd8, 0xe3,
0x8d, 0xc0, 0xb4, 0xe7, 0xf6, 0x78, 0x23, 0x04, 0x79, 0xfc, 0xe1, 0xe1, 0x2f, 0x72, 0x30, 0x1d,
0x30, 0xf7, 0xfd, 0x78, 0x23, 0x45, 0xe4, 0x77, 0x8f, 0x60, 0x7b, 0x3f, 0x82, 0xfd, 0x52, 0x82,
0xf1, 0xc0, 0x75, 0xbf, 0x79, 0x0f, 0x2a, 0x02, 0xdb, 0x32, 0x5a, 0xbc, 0xff, 0xce, 0x85, 0x27,
0xf0, 0x5b, 0x7f, 0xab, 0xff, 0xc3, 0x5f, 0xae, 0xca, 0x5f, 0xe7, 0x61, 0x32, 0xbe, 0x1b, 0x23,
0x97, 0xbf, 0x52, 0xcf, 0xcb, 0xdf, 0x0d, 0x38, 0xb7, 0xe3, 0x68, 0x5a, 0x97, 0xbb, 0x21, 0x74,
0x03, 0xec, 0x5e, 0xde, 0xbc, 0x22, 0x24, 0xcf, 0xdd, 0x4b, 0xe1, 0xc1, 0xa9, 0x92, 0x19, 0x17,
0xd9, 0xf9, 0x13, 0x5d, 0x64, 0x27, 0xee, 0x55, 0x0b, 0x03, 0xdc, 0xab, 0xa6, 0x5e, 0x4a, 0x0f,
0x9d, 0xe0, 0x52, 0xfa, 0x24, 0xb7, 0xc8, 0x29, 0x49, 0xac, 0xe7, 0xa3, 0xc6, 0x57, 0x60, 0x4e,
0x88, 0x51, 0x7e, 0xc1, 0xab, 0x53, 0xcb, 0xd0, 0x34, 0x62, 0x2d, 0x39, 0x9d, 0x4e, 0x57, 0x7e,
0x17, 0xc6, 0xa3, 0x4f, 0x17, 0xdc, 0x95, 0x76, 0x5f, 0x4f, 0x88, 0x2b, 0xb4, 0xd0, 0x4a, 0xbb,
0xe3, 0xd8, 0xe7, 0x90, 0xff, 0x46, 0x82, 0x99, 0xf4, 0x27, 0x8a, 0x48, 0x83, 0xf1, 0x8e, 0x72,
0x10, 0x7e, 0x36, 0x2a, 0x9d, 0xf0, 0x70, 0x83, 0xdf, 0x59, 0xaf, 0x46, 0xb0, 0x70, 0x0c, 0x5b,
0xfe, 0x5e, 0x82, 0xd9, 0x8c, 0xdb, 0xe2, 0xd3, 0xb5, 0x04, 0x7d, 0x08, 0xa5, 0x8e, 0x72, 0xd0,
0x70, 0xac, 0x36, 0x39, 0xf1, 0x71, 0x0e, 0xcf, 0x18, 0xab, 0x02, 0x05, 0xfb, 0x78, 0xf2, 0xe7,
0x12, 0x94, 0xb3, 0x1a, 0x6b, 0x74, 0x33, 0x72, 0xaf, 0xfd, 0x6a, 0xec, 0x5e, 0x7b, 0x2a, 0x21,
0xf7, 0x82, 0x6e, 0xb5, 0xff, 0x4b, 0x82, 0x99, 0xf4, 0x0f, 0x0c, 0xf4, 0x66, 0xc4, 0xc2, 0x4a,
0xcc, 0xc2, 0x89, 0x98, 0x94, 0xb0, 0xef, 0x23, 0x18, 0x17, 0x9f, 0x21, 0x02, 0x46, 0x78, 0x55,
0x4e, 0xcb, 0x95, 0x02, 0xc2, 0x6b, 0xbb, 0xf9, 0x7a, 0x45, 0xc7, 0x70, 0x0c, 0x4d, 0xfe, 0xdb,
0x1c, 0x0c, 0x35, 0x9a, 0x8a, 0x46, 0x4e, 0xa1, 0xcd, 0x7a, 0x3f, 0xd2, 0x66, 0xf5, 0xfa, 0x17,
0x0f, 0x6e, 0x55, 0x66, 0x87, 0x85, 0x63, 0x1d, 0xd6, 0xeb, 0x7d, 0xa1, 0x1d, 0xdf, 0x5c, 0xfd,
0x01, 0x8c, 0xf8, 0x4a, 0x07, 0xcb, 0xf9, 0xf2, 0xbf, 0xe7, 0x60, 0x34, 0xa4, 0x62, 0xc0, 0x8a,
0xb1, 0x13, 0xa9, 0xb4, 0xfd, 0xfc, 0x63, 0x5d, 0x48, 0x57, 0xd5, 0xab, 0xad, 0xee, 0x13, 0xc5,
0xe0, 0x51, 0x5a, 0xb2, 0xe4, 0xbe, 0x0b, 0xe3, 0x94, 0xff, 0xe3, 0x99, 0x7f, 0x08, 0x9a, 0xe7,
0xb1, 0xe8, 0x3f, 0x6c, 0xdd, 0x8c, 0x50, 0x71, 0x8c, 0x7b, 0xee, 0x0e, 0x8c, 0x45, 0x94, 0x0d,
0xf4, 0xc2, 0xf0, 0x7f, 0x25, 0x78, 0xb5, 0xe7, 0x27, 0x2a, 0xaa, 0x47, 0x36, 0x49, 0x35, 0xb6,
0x49, 0xe6, 0xb3, 0x01, 0x5e, 0xdc, 0x4b, 0x95, 0xfa, 0xb5, 0xa7, 0xdf, 0xcd, 0x9f, 0xf9, 0xe6,
0xbb, 0xf9, 0x33, 0xdf, 0x7e, 0x37, 0x7f, 0xe6, 0xaf, 0x8e, 0xe6, 0xa5, 0xa7, 0x47, 0xf3, 0xd2,
0x37, 0x47, 0xf3, 0xd2, 0xb7, 0x47, 0xf3, 0xd2, 0xcf, 0x8e, 0xe6, 0xa5, 0x7f, 0xfc, 0x7e, 0xfe,
0xcc, 0x87, 0x45, 0x01, 0xf7, 0xeb, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0xb3, 0xc8, 0xe2, 0x54,
0x3c, 0x00, 0x00,
}

12
staging/src/k8s.io/api/extensions/v1beta1/generated.proto
View File

@@ -32,6 +32,12 @@ import "k8s.io/apimachinery/pkg/util/intstr/generated.proto";
// Package-wide variables from generator "generated".
option go_package = "v1beta1";
// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
message AllowedFlexVolume {
// Driver is the name of the Flexvolume driver.
optional string driver = 1;
}
// defines the host volume conditions that will be enabled by a policy
// for pods to use. It requires the path prefix to be defined.
message AllowedHostPath {
@@ -843,6 +849,12 @@ message PodSecurityPolicySpec {
// is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
repeated AllowedHostPath allowedHostPaths = 17;
// AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
// Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
// is allowed in the "Volumes" field.
// +optional
repeated AllowedFlexVolume allowedFlexVolumes = 18;
}
// DEPRECATED - This group version of ReplicaSet is deprecated by apps/v1beta2/ReplicaSet. See the release notes for

11
staging/src/k8s.io/api/extensions/v1beta1/types.go
View File

@@ -938,6 +938,11 @@ type PodSecurityPolicySpec struct {
// is a white list of allowed host paths. Empty indicates that all host paths may be used.
// +optional
AllowedHostPaths []AllowedHostPath `json:"allowedHostPaths,omitempty" protobuf:"bytes,17,rep,name=allowedHostPaths"`
// AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all
// Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes
// is allowed in the "Volumes" field.
// +optional
AllowedFlexVolumes []AllowedFlexVolume `json:"allowedFlexVolumes,omitempty" protobuf:"bytes,18,rep,name=allowedFlexVolumes"`
}
// defines the host volume conditions that will be enabled by a policy
@@ -981,6 +986,12 @@ var (
All FSType = "*"
)
// AllowedFlexVolume represents a single Flexvolume that is allowed to be used.
type AllowedFlexVolume struct {
// Driver is the name of the Flexvolume driver.
Driver string `json:"driver" protobuf:"bytes,1,opt,name=driver"`
}
// Host Port Range defines a range of host ports that will be enabled by a policy
// for pods to use. It requires both the start and end to be defined.
type HostPortRange struct {

10
staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go generated
View File

@@ -27,6 +27,15 @@ package v1beta1
// Those methods can be generated by using hack/update-generated-swagger-docs.sh
// AUTO-GENERATED FUNCTIONS START HERE
var map_AllowedFlexVolume = map[string]string{
"": "AllowedFlexVolume represents a single Flexvolume that is allowed to be used.",
"driver": "Driver is the name of the Flexvolume driver.",
}
func (AllowedFlexVolume) SwaggerDoc() map[string]string {
return map_AllowedFlexVolume
}
var map_AllowedHostPath = map[string]string{
"": "defines the host volume conditions that will be enabled by a policy for pods to use. It requires the path prefix to be defined.",
"pathPrefix": "is the path prefix that the host volume must match. It does not support `*`. Trailing slashes are trimmed when validating the path prefix with a host path.\n\nExamples: `/foo` would allow `/foo`, `/foo/` and `/foo/bar` `/foo` would not allow `/food` or `/etc/foo`",
@@ -465,6 +474,7 @@ var map_PodSecurityPolicySpec = map[string]string{
"defaultAllowPrivilegeEscalation": "DefaultAllowPrivilegeEscalation controls the default setting for whether a process can gain more privileges than its parent process.",
"allowPrivilegeEscalation": "AllowPrivilegeEscalation determines if a pod can request to allow privilege escalation. If unspecified, defaults to true.",
"allowedHostPaths": "is a white list of allowed host paths. Empty indicates that all host paths may be used.",
"allowedFlexVolumes": "AllowedFlexVolumes is a whitelist of allowed Flexvolumes. Empty or nil indicates that all Flexvolumes may be used. This parameter is effective only when the usage of the Flexvolumes is allowed in the \"Volumes\" field.",
}
func (PodSecurityPolicySpec) SwaggerDoc() map[string]string {

21
staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go generated
View File

@@ -27,6 +27,22 @@ import (
intstr "k8s.io/apimachinery/pkg/util/intstr"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedFlexVolume) DeepCopyInto(out *AllowedFlexVolume) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllowedFlexVolume.
func (in *AllowedFlexVolume) DeepCopy() *AllowedFlexVolume {
if in == nil {
return nil
}
out := new(AllowedFlexVolume)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllowedHostPath) DeepCopyInto(out *AllowedHostPath) {
*out = *in
@@ -1194,6 +1210,11 @@ func (in *PodSecurityPolicySpec) DeepCopyInto(out *PodSecurityPolicySpec) {
*out = make([]AllowedHostPath, len(*in))
copy(*out, *in)
}
if in.AllowedFlexVolumes != nil {
in, out := &in.AllowedFlexVolumes, &out.AllowedFlexVolumes
*out = make([]AllowedFlexVolume, len(*in))
copy(*out, *in)
}
return
}

6
staging/src/k8s.io/client-go/listers/apps/v1/BUILD
View File

@@ -5,16 +5,22 @@ go_library(
srcs = [
"controllerrevision.go",
"daemonset.go",
"daemonset_expansion.go",
"deployment.go",
"deployment_expansion.go",
"expansion_generated.go",
"replicaset.go",
"replicaset_expansion.go",
"statefulset.go",
"statefulset_expansion.go",
],
importpath = "k8s.io/client-go/listers/apps/v1",
visibility = ["//visibility:public"],
deps = [
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],

113
staging/src/k8s.io/client-go/listers/apps/v1/daemonset_expansion.go Normal file
View File

@@ -0,0 +1,113 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
"fmt"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
// DaemonSetListerExpansion allows custom methods to be added to
// DaemonSetLister.
type DaemonSetListerExpansion interface {
GetPodDaemonSets(pod *v1.Pod) ([]*apps.DaemonSet, error)
GetHistoryDaemonSets(history *apps.ControllerRevision) ([]*apps.DaemonSet, error)
}
// DaemonSetNamespaceListerExpansion allows custom methods to be added to
// DaemonSetNamespaceLister.
type DaemonSetNamespaceListerExpansion interface{}
// GetPodDaemonSets returns a list of DaemonSets that potentially match a pod.
// Only the one specified in the Pod's ControllerRef will actually manage it.
// Returns an error only if no matching DaemonSets are found.
func (s *daemonSetLister) GetPodDaemonSets(pod *v1.Pod) ([]*apps.DaemonSet, error) {
var selector labels.Selector
var daemonSet *apps.DaemonSet
if len(pod.Labels) == 0 {
return nil, fmt.Errorf("no daemon sets found for pod %v because it has no labels", pod.Name)
}
list, err := s.DaemonSets(pod.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
var daemonSets []*apps.DaemonSet
for i := range list {
daemonSet = list[i]
if daemonSet.Namespace != pod.Namespace {
continue
}
selector, err = metav1.LabelSelectorAsSelector(daemonSet.Spec.Selector)
if err != nil {
// this should not happen if the DaemonSet passed validation
return nil, err
}
// If a daemonSet with a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
continue
}
daemonSets = append(daemonSets, daemonSet)
}
if len(daemonSets) == 0 {
return nil, fmt.Errorf("could not find daemon set for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
}
return daemonSets, nil
}
// GetHistoryDaemonSets returns a list of DaemonSets that potentially
// match a ControllerRevision. Only the one specified in the ControllerRevision's ControllerRef
// will actually manage it.
// Returns an error only if no matching DaemonSets are found.
func (s *daemonSetLister) GetHistoryDaemonSets(history *apps.ControllerRevision) ([]*apps.DaemonSet, error) {
if len(history.Labels) == 0 {
return nil, fmt.Errorf("no DaemonSet found for ControllerRevision %s because it has no labels", history.Name)
}
list, err := s.DaemonSets(history.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
var daemonSets []*apps.DaemonSet
for _, ds := range list {
selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("invalid label selector: %v", err)
}
// If a DaemonSet with a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() || !selector.Matches(labels.Set(history.Labels)) {
continue
}
daemonSets = append(daemonSets, ds)
}
if len(daemonSets) == 0 {
return nil, fmt.Errorf("could not find DaemonSets for ControllerRevision %s in namespace %s with labels: %v", history.Name, history.Namespace, history.Labels)
}
return daemonSets, nil
}

70
staging/src/k8s.io/client-go/listers/apps/v1/deployment_expansion.go Normal file
View File

@@ -0,0 +1,70 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
"fmt"
apps "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
// DeploymentListerExpansion allows custom methods to be added to
// DeploymentLister.
type DeploymentListerExpansion interface {
GetDeploymentsForReplicaSet(rs *apps.ReplicaSet) ([]*apps.Deployment, error)
}
// DeploymentNamespaceListerExpansion allows custom methods to be added to
// DeploymentNamespaceLister.
type DeploymentNamespaceListerExpansion interface{}
// GetDeploymentsForReplicaSet returns a list of Deployments that potentially
// match a ReplicaSet. Only the one specified in the ReplicaSet's ControllerRef
// will actually manage it.
// Returns an error only if no matching Deployments are found.
func (s *deploymentLister) GetDeploymentsForReplicaSet(rs *apps.ReplicaSet) ([]*apps.Deployment, error) {
if len(rs.Labels) == 0 {
return nil, fmt.Errorf("no deployments found for ReplicaSet %v because it has no labels", rs.Name)
}
// TODO: MODIFY THIS METHOD so that it checks for the podTemplateSpecHash label
dList, err := s.Deployments(rs.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
var deployments []*apps.Deployment
for _, d := range dList {
selector, err := metav1.LabelSelectorAsSelector(d.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("invalid label selector: %v", err)
}
// If a deployment with a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() || !selector.Matches(labels.Set(rs.Labels)) {
continue
}
deployments = append(deployments, d)
}
if len(deployments) == 0 {
return nil, fmt.Errorf("could not find deployments set for ReplicaSet %s in namespace %s with labels: %v", rs.Name, rs.Namespace, rs.Labels)
}
return deployments, nil
}

32
staging/src/k8s.io/client-go/listers/apps/v1/expansion_generated.go
View File

@@ -25,35 +25,3 @@ type ControllerRevisionListerExpansion interface{}
// ControllerRevisionNamespaceListerExpansion allows custom methods to be added to
// ControllerRevisionNamespaceLister.
type ControllerRevisionNamespaceListerExpansion interface{}
// DaemonSetListerExpansion allows custom methods to be added to
// DaemonSetLister.
type DaemonSetListerExpansion interface{}
// DaemonSetNamespaceListerExpansion allows custom methods to be added to
// DaemonSetNamespaceLister.
type DaemonSetNamespaceListerExpansion interface{}
// DeploymentListerExpansion allows custom methods to be added to
// DeploymentLister.
type DeploymentListerExpansion interface{}
// DeploymentNamespaceListerExpansion allows custom methods to be added to
// DeploymentNamespaceLister.
type DeploymentNamespaceListerExpansion interface{}
// ReplicaSetListerExpansion allows custom methods to be added to
// ReplicaSetLister.
type ReplicaSetListerExpansion interface{}
// ReplicaSetNamespaceListerExpansion allows custom methods to be added to
// ReplicaSetNamespaceLister.
type ReplicaSetNamespaceListerExpansion interface{}
// StatefulSetListerExpansion allows custom methods to be added to
// StatefulSetLister.
type StatefulSetListerExpansion interface{}
// StatefulSetNamespaceListerExpansion allows custom methods to be added to
// StatefulSetNamespaceLister.
type StatefulSetNamespaceListerExpansion interface{}

73
staging/src/k8s.io/client-go/listers/apps/v1/replicaset_expansion.go Normal file
View File

@@ -0,0 +1,73 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
"fmt"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
// ReplicaSetListerExpansion allows custom methods to be added to
// ReplicaSetLister.
type ReplicaSetListerExpansion interface {
GetPodReplicaSets(pod *v1.Pod) ([]*apps.ReplicaSet, error)
}
// ReplicaSetNamespaceListerExpansion allows custom methods to be added to
// ReplicaSetNamespaceLister.
type ReplicaSetNamespaceListerExpansion interface{}
// GetPodReplicaSets returns a list of ReplicaSets that potentially match a pod.
// Only the one specified in the Pod's ControllerRef will actually manage it.
// Returns an error only if no matching ReplicaSets are found.
func (s *replicaSetLister) GetPodReplicaSets(pod *v1.Pod) ([]*apps.ReplicaSet, error) {
if len(pod.Labels) == 0 {
return nil, fmt.Errorf("no ReplicaSets found for pod %v because it has no labels", pod.Name)
}
list, err := s.ReplicaSets(pod.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
var rss []*apps.ReplicaSet
for _, rs := range list {
if rs.Namespace != pod.Namespace {
continue
}
selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("invalid selector: %v", err)
}
// If a ReplicaSet with a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
continue
}
rss = append(rss, rs)
}
if len(rss) == 0 {
return nil, fmt.Errorf("could not find ReplicaSet for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
}
return rss, nil
}

77
staging/src/k8s.io/client-go/listers/apps/v1/statefulset_expansion.go Normal file
View File

@@ -0,0 +1,77 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
"fmt"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
// StatefulSetListerExpansion allows custom methods to be added to
// StatefulSetLister.
type StatefulSetListerExpansion interface {
GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error)
}
// StatefulSetNamespaceListerExpansion allows custom methods to be added to
// StatefulSetNamespaceLister.
type StatefulSetNamespaceListerExpansion interface{}
// GetPodStatefulSets returns a list of StatefulSets that potentially match a pod.
// Only the one specified in the Pod's ControllerRef will actually manage it.
// Returns an error only if no matching StatefulSets are found.
func (s *statefulSetLister) GetPodStatefulSets(pod *v1.Pod) ([]*apps.StatefulSet, error) {
var selector labels.Selector
var ps *apps.StatefulSet
if len(pod.Labels) == 0 {
return nil, fmt.Errorf("no StatefulSets found for pod %v because it has no labels", pod.Name)
}
list, err := s.StatefulSets(pod.Namespace).List(labels.Everything())
if err != nil {
return nil, err
}
var psList []*apps.StatefulSet
for i := range list {
ps = list[i]
if ps.Namespace != pod.Namespace {
continue
}
selector, err = metav1.LabelSelectorAsSelector(ps.Spec.Selector)
if err != nil {
return nil, fmt.Errorf("invalid selector: %v", err)
}
// If a StatefulSet with a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() || !selector.Matches(labels.Set(pod.Labels)) {
continue
}
psList = append(psList, ps)
}
if len(psList) == 0 {
return nil, fmt.Errorf("could not find StatefulSet for pod %s in namespace %s with labels: %v", pod.Name, pod.Namespace, pod.Labels)
}
return psList, nil
}

2
test/e2e/scalability/density.go
View File

@@ -200,7 +200,7 @@ func density30AddonResourceVerifier(numNodes int) map[string]framework.ResourceC
MemoryConstraint: 100 * (1024 * 1024),
}
constraints["l7-lb-controller"] = framework.ResourceConstraint{
CPUConstraint: 0.2 + 0.0001*float64(numNodes),
CPUConstraint: 0.2 + 0.00015*float64(numNodes),
MemoryConstraint: (75 + uint64(math.Ceil(0.8*float64(numNodes)))) * (1024 * 1024),
}
constraints["influxdb"] = framework.ResourceConstraint{