diff --git a/CHANGELOG-1.12.md b/CHANGELOG-1.12.md index 61128f607be..7658a50044e 100644 --- a/CHANGELOG-1.12.md +++ b/CHANGELOG-1.12.md @@ -1,66 +1,73 @@ -- [v1.12.8](#v1128) - - [Downloads for v1.12.8](#downloads-for-v1128) +- [v1.12.9](#v1129) + - [Downloads for v1.12.9](#downloads-for-v1129) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.12.7](#changelog-since-v1127) + - [Changelog since v1.12.8](#changelog-since-v1128) - [Other notable changes](#other-notable-changes) -- [v1.12.7](#v1127) - - [Downloads for v1.12.7](#downloads-for-v1127) +- [v1.12.8](#v1128) + - [Downloads for v1.12.8](#downloads-for-v1128) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.12.6](#changelog-since-v1126) + - [Changelog since v1.12.7](#changelog-since-v1127) - [Other notable changes](#other-notable-changes-1) -- [v1.12.6](#v1126) - - [Downloads for v1.12.6](#downloads-for-v1126) +- [v1.12.7](#v1127) + - [Downloads for v1.12.7](#downloads-for-v1127) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.12.5](#changelog-since-v1125) + - [Changelog since v1.12.6](#changelog-since-v1126) - [Other notable changes](#other-notable-changes-2) -- [v1.12.5](#v1125) - - [Downloads for v1.12.5](#downloads-for-v1125) +- [v1.12.6](#v1126) + - [Downloads for v1.12.6](#downloads-for-v1126) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - - [Changelog since v1.12.4](#changelog-since-v1124) + - [Changelog since v1.12.5](#changelog-since-v1125) - [Other notable changes](#other-notable-changes-3) -- [v1.12.4](#v1124) - - [Downloads for v1.12.4](#downloads-for-v1124) +- [v1.12.5](#v1125) + - [Downloads for v1.12.5](#downloads-for-v1125) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - - [Changelog since v1.12.3](#changelog-since-v1123) - - [Action Required](#action-required) + - [Changelog since v1.12.4](#changelog-since-v1124) - [Other notable changes](#other-notable-changes-4) -- [v1.12.3](#v1123) - - [Downloads for v1.12.3](#downloads-for-v1123) +- [v1.12.4](#v1124) + - [Downloads for v1.12.4](#downloads-for-v1124) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - - [Changelog since v1.12.2](#changelog-since-v1122) + - [Changelog since v1.12.3](#changelog-since-v1123) + - [Action Required](#action-required) - [Other notable changes](#other-notable-changes-5) -- [v1.12.2](#v1122) - - [Downloads for v1.12.2](#downloads-for-v1122) +- [v1.12.3](#v1123) + - [Downloads for v1.12.3](#downloads-for-v1123) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - - [Changelog since v1.12.1](#changelog-since-v1121) + - [Changelog since v1.12.2](#changelog-since-v1122) - [Other notable changes](#other-notable-changes-6) -- [v1.12.1](#v1121) - - [Downloads for v1.12.1](#downloads-for-v1121) +- [v1.12.2](#v1122) + - [Downloads for v1.12.2](#downloads-for-v1122) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) - - [Changelog since v1.12.0](#changelog-since-v1120) + - [Changelog since v1.12.1](#changelog-since-v1121) - [Other notable changes](#other-notable-changes-7) -- [v1.12.0](#v1120) - - [Downloads for v1.12.0](#downloads-for-v1120) +- [v1.12.1](#v1121) + - [Downloads for v1.12.1](#downloads-for-v1121) - [Client Binaries](#client-binaries-8) - [Server Binaries](#server-binaries-8) - [Node Binaries](#node-binaries-8) + - [Changelog since v1.12.0](#changelog-since-v1120) + - [Other notable changes](#other-notable-changes-8) +- [v1.12.0](#v1120) + - [Downloads for v1.12.0](#downloads-for-v1120) + - [Client Binaries](#client-binaries-9) + - [Server Binaries](#server-binaries-9) + - [Node Binaries](#node-binaries-9) - [Known Issues](#known-issues) - [Major Themes](#major-themes) - [SIG API Machinery](#sig-api-machinery) @@ -82,7 +89,7 @@ - [Deprecations and removals](#deprecations-and-removals) - [New Features](#new-features) - [API Changes](#api-changes) - - [Other Notable Changes](#other-notable-changes-8) + - [Other Notable Changes](#other-notable-changes-9) - [SIG API Machinery](#sig-api-machinery-1) - [SIG Apps](#sig-apps) - [SIG Auth](#sig-auth) @@ -101,54 +108,126 @@ - [SIG Storage](#sig-storage-1) - [SIG VMWare](#sig-vmware-1) - [SIG Windows](#sig-windows-1) - - [Other Notable Changes](#other-notable-changes-9) + - [Other Notable Changes](#other-notable-changes-10) - [Bug Fixes](#bug-fixes) - [Not Very Notable (that is, non-user-facing)](#not-very-notable-that-is-non-user-facing) - [External Dependencies](#external-dependencies) - [v1.12.0-rc.2](#v1120-rc2) - [Downloads for v1.12.0-rc.2](#downloads-for-v1120-rc2) - - [Client Binaries](#client-binaries-9) - - [Server Binaries](#server-binaries-9) - - [Node Binaries](#node-binaries-9) - - [Changelog since v1.12.0-rc.1](#changelog-since-v1120-rc1) - - [Other notable changes](#other-notable-changes-10) -- [v1.12.0-rc.1](#v1120-rc1) - - [Downloads for v1.12.0-rc.1](#downloads-for-v1120-rc1) - [Client Binaries](#client-binaries-10) - [Server Binaries](#server-binaries-10) - [Node Binaries](#node-binaries-10) - - [Changelog since v1.12.0-beta.2](#changelog-since-v1120-beta2) - - [Action Required](#action-required-2) + - [Changelog since v1.12.0-rc.1](#changelog-since-v1120-rc1) - [Other notable changes](#other-notable-changes-11) -- [v1.12.0-beta.2](#v1120-beta2) - - [Downloads for v1.12.0-beta.2](#downloads-for-v1120-beta2) +- [v1.12.0-rc.1](#v1120-rc1) + - [Downloads for v1.12.0-rc.1](#downloads-for-v1120-rc1) - [Client Binaries](#client-binaries-11) - [Server Binaries](#server-binaries-11) - [Node Binaries](#node-binaries-11) - - [Changelog since v1.12.0-beta.1](#changelog-since-v1120-beta1) - - [Action Required](#action-required-3) + - [Changelog since v1.12.0-beta.2](#changelog-since-v1120-beta2) + - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-12) -- [v1.12.0-beta.1](#v1120-beta1) - - [Downloads for v1.12.0-beta.1](#downloads-for-v1120-beta1) +- [v1.12.0-beta.2](#v1120-beta2) + - [Downloads for v1.12.0-beta.2](#downloads-for-v1120-beta2) - [Client Binaries](#client-binaries-12) - [Server Binaries](#server-binaries-12) - [Node Binaries](#node-binaries-12) - - [Changelog since v1.12.0-alpha.1](#changelog-since-v1120-alpha1) - - [Action Required](#action-required-4) + - [Changelog since v1.12.0-beta.1](#changelog-since-v1120-beta1) + - [Action Required](#action-required-3) - [Other notable changes](#other-notable-changes-13) -- [v1.12.0-alpha.1](#v1120-alpha1) - - [Downloads for v1.12.0-alpha.1](#downloads-for-v1120-alpha1) +- [v1.12.0-beta.1](#v1120-beta1) + - [Downloads for v1.12.0-beta.1](#downloads-for-v1120-beta1) - [Client Binaries](#client-binaries-13) - [Server Binaries](#server-binaries-13) - [Node Binaries](#node-binaries-13) + - [Changelog since v1.12.0-alpha.1](#changelog-since-v1120-alpha1) + - [Action Required](#action-required-4) + - [Other notable changes](#other-notable-changes-14) +- [v1.12.0-alpha.1](#v1120-alpha1) + - [Downloads for v1.12.0-alpha.1](#downloads-for-v1120-alpha1) + - [Client Binaries](#client-binaries-14) + - [Server Binaries](#server-binaries-14) + - [Node Binaries](#node-binaries-14) - [Changelog since v1.11.0](#changelog-since-v1110) - [Action Required](#action-required-5) - - [Other notable changes](#other-notable-changes-14) + - [Other notable changes](#other-notable-changes-15) +# v1.12.9 + +[Documentation](https://docs.k8s.io) + +## Downloads for v1.12.9 + + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes.tar.gz) | `27482e1704256927b2c494e933e5e481280350eddbf4858ab8bbf980784630c295e9b8a882e363e2e619439c3636a849b95a010eb55dc98f73b19e37d3d4ea2e` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-src.tar.gz) | `654ea2da90e61f9c5f962f2131af2d46452d2f7f629c87edcacdd3b197c0e2ea83fed341cebcfffe4c47df42ce70b6709254517215960fbde18443c43692d4fe` + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-darwin-386.tar.gz) | `809a9e225234cb37748c4f33d8028ca8ac52e12f4459ee710e4969230adf90340b7c245b835b8108e16333b0e159aa108dae1a4a9f80fb9d1c3c6220ddc59b97` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-darwin-amd64.tar.gz) | `660b1ee830c75d0a8b2be611cea0c1fdbd922895f4bfc714d66345c214b63e72528c873b337c03102d7a216df01de98cfd9c74f176172e165c120b7199e22687` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-386.tar.gz) | `c663732322edb13f3958766c9e50197d654abe89ce8ca943f35948bd083f89b38b5a59561fac583b826e445b399d7ad2280eb2ee55824b5e5674816d80f799f5` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-amd64.tar.gz) | `e1c4945644e2434d0938a4a565672143156ceb517618a695f84e92a8bc04f514491323c54e4e53762ad8c6199578a6600e976b2010f27e2feb006227da3d1235` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-arm.tar.gz) | `fe9d040544b0880475834d57a5bc4eb2865d391650076ab86d0c73c8d76348c357a083f9eff60e100bf9e40a63a5dd489c17a17e85efda12f6b23991821a73bd` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-arm64.tar.gz) | `26d8b699a663f3bd9ffc43b32861a4903f714e2a147c75d2407c0c479bea97ba5fdeeb7570f1554df5be939f5f113d3d2f0c1ca5b3f4a5a7e551cc1ecc9b22ad` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-ppc64le.tar.gz) | `c4b0f62c3b6418c2efc85c57355461342c97c8032a61d3aa5952cb63e62dd7032546c1a936399e89531bf3458377bd20810439b49fe77a1642bbf0023d29113e` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-linux-s390x.tar.gz) | `c3ce8e29c6c144e203c15c04414f907a68d89089a8e7f451f80cc2507665abcbfd8ecdedccec466644036ca5612aea5b4113a62f558bcb827c64a928d736fb27` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-windows-386.tar.gz) | `f62e08eba18a94867a595ed5c4256b250cc8fe3a87de9dd1ca8c455704070d17f47a80a11a802fdf94ab61bc7df1107476d13f785dc21beb00db83969ac201b7` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-client-windows-amd64.tar.gz) | `0b655afcf05e54c6eb78a3a58f5713a09672a11915129d769e4a1c0d8c6b5ae6301f58efb7a65b115c04d74b578a7cc81b16ba91f37635493cf0435495d71835` + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-server-linux-amd64.tar.gz) | `fb5a0f5cd8c06fd8178affe081118db4f11e618c40be251f4348ea241fdde35bec3fdabeb1ac0e99056f64cd13d99715edfbd44ff197072185793cbe23badbdc` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-server-linux-arm.tar.gz) | `5c998f415ea8e1b96385d557aca46691f041e98bafa5ad8a4e110f60155b05106dcf313256c7622819145be44edf897a789fa7304ccb4e1df634071d37de3ad1` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-server-linux-arm64.tar.gz) | `9501823b79673b129a7abb5f07700259faee10849da710bf12979468b44e6fda2e93e7f2e77db913edcdad4477580d398cbe3b24eca1cc0f0731c407357b8e4c` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-server-linux-ppc64le.tar.gz) | `c4d814fc498923f257c0a96e48d0adbea2487308f269c0e70118de89df4b557b523d507d48bb2432f74235b4461f3a50fa26d5bbee1e020436d5c6591ae88958` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-server-linux-s390x.tar.gz) | `dfb06bb352db236ea6763d0237bb1cf6a26c2d04c54b85de19dd6bc096ce93259038123049a32d0efc982e559628ed27cb41c56d2bf7f46b13a8687e4d478fb0` + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-linux-amd64.tar.gz) | `32017a8f8d47bf4f8ac9507f278bbc814d70a804d1d9b27ffd4ae4c2b9646b34513b7ea972839dabc8d83e0be67f32b0baa4b77a4f2eaefa56ba5f6d917fe1a0` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-linux-arm.tar.gz) | `cb2b3563806dc488a2e8b9d54d545b3f4cbad746d38e3d847d5ab5a1b59af10183dc72835fc22557416d2c2722afa4b37083c1000edc1cd522ce144580c2746e` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-linux-arm64.tar.gz) | `e1bd333c3bd8bad52af1c696eb28ffcc058ba5fd6c554244627de8d9231b69466e33a40132b3e03158dd57b61c2080eecd559ec295c2db91b6340dee625b0277` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-linux-ppc64le.tar.gz) | `ecf696b8522e1dffa61e00aa3e27aad27135487640f8c58af84ca883827ad568ec96e9eb4ccd2220e43bb3c1afae9da639047d21e545c8fa9a17ffa66b2a130a` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-linux-s390x.tar.gz) | `e6177729ce9aadc31dd8237f7282cbbe30f6432ab14931310eb6fe892edb126e4ac6e5212b2aa8d2cfd1f00af228dcb1303faf4cc25a0a9abf3cee7892b9cda7` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.12.9/kubernetes-node-windows-amd64.tar.gz) | `c2daa564e89d6ec8c5322e3b2b7188ac1cb1091ab7f56121f3017ae8ea334e4b626978b31dbed31f7e9b2b1e8e4ada7bb8376831fb61e364568296041bf39fe0` + +## Changelog since v1.12.8 + +### Other notable changes + +* Active watches of custom resources now terminate properly if the CRD is modified. ([#78029](https://github.com/kubernetes/kubernetes/pull/78029), [@liggitt](https://github.com/liggitt)) +* fix incorrect prometheus azure metrics ([#77722](https://github.com/kubernetes/kubernetes/pull/77722), [@andyzhangx](https://github.com/andyzhangx)) +* Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion. ([#77619](https://github.com/kubernetes/kubernetes/pull/77619), [@caesarxuchao](https://github.com/caesarxuchao)) +* client-go and kubectl no longer write cached discovery files with world-accessible file permissions ([#77874](https://github.com/kubernetes/kubernetes/pull/77874), [@yuchengwu](https://github.com/yuchengwu)) +* Check if container memory stats are available before accessing it ([#77656](https://github.com/kubernetes/kubernetes/pull/77656), [@yastij](https://github.com/yastij)) +* Fixes segmentation fault issue with Protobuf library when log entries are deeply nested. ([#77224](https://github.com/kubernetes/kubernetes/pull/77224), [@qingling128](https://github.com/qingling128)) +* Clean links handling in cp's tar code ([#76788](https://github.com/kubernetes/kubernetes/pull/76788), [@soltysh](https://github.com/soltysh)) +* [fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.2 to pick up security fixes. ([#76762](https://github.com/kubernetes/kubernetes/pull/76762), [@serathius](https://github.com/serathius)) +* Fixes an error with stuck informers when an etcd watch receives update or delete events with missing data ([#76675](https://github.com/kubernetes/kubernetes/pull/76675), [@ryanmcnamara](https://github.com/ryanmcnamara)) +* fix azure disk list corruption issue ([#77187](https://github.com/kubernetes/kubernetes/pull/77187), [@andyzhangx](https://github.com/andyzhangx)) +* Fixed scanning of failed iSCSI targets. ([#74306](https://github.com/kubernetes/kubernetes/pull/74306), [@jsafrane](https://github.com/jsafrane)) +* fix detach azure disk back off issue which has too big lock in failure retry condition ([#76573](https://github.com/kubernetes/kubernetes/pull/76573), [@andyzhangx](https://github.com/andyzhangx)) +* specify azure file share name in azure file plugin ([#76988](https://github.com/kubernetes/kubernetes/pull/76988), [@andyzhangx](https://github.com/andyzhangx)) +* [metrics-server addon] Restore connecting to nodes via IP addresses ([#76819](https://github.com/kubernetes/kubernetes/pull/76819), [@serathius](https://github.com/serathius)) +* Update Cluster Autoscaler to 1.12.5 ([#77063](https://github.com/kubernetes/kubernetes/pull/77063), [@losipiuk](https://github.com/losipiuk)) + * - https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.12.5 + * - https://github.com/kubernetes/autoscaler/releases/tag/cluster-autoscaler-1.12.4 + + + # v1.12.8 [Documentation](https://docs.k8s.io) diff --git a/CHANGELOG-1.14.md b/CHANGELOG-1.14.md index 017d5cc8502..5fdb4942738 100644 --- a/CHANGELOG-1.14.md +++ b/CHANGELOG-1.14.md @@ -967,12 +967,7 @@ filename | sha512 hash * While this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable. * Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified. ([#74562](https://github.com/kubernetes/kubernetes/pull/74562), [@roycaihw](https://github.com/roycaihw)) * client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder ([#73676](https://github.com/kubernetes/kubernetes/pull/73676), [@martin-helmich](https://github.com/martin-helmich)) -* # Apply resources from a directory containing kustomization.yaml ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1)) - * kubectl apply -k dir - * # Delete resources from a directory containing kustomization.yaml. - * kubectl delete -k dir - * # List resources from a directory containing kustomization.yaml - * kubectl get -k dir +* The examples in kubectl apply/get/delete are updated to support `-k` which uses a `kustomization.yaml` file. ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1)) * kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments. ([#74168](https://github.com/kubernetes/kubernetes/pull/74168), [@ereslibre](https://github.com/ereslibre)) * Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks ([#74448](https://github.com/kubernetes/kubernetes/pull/74448), [@liggitt](https://github.com/liggitt)) * client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`) ([#74422](https://github.com/kubernetes/kubernetes/pull/74422), [@liggitt](https://github.com/liggitt)) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 9deca196401..95c9c24ccd2 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -11222,6 +11222,16 @@ }, "io.k8s.api.core.v1.WindowsSecurityContextOptions": { "description": "WindowsSecurityContextOptions contain Windows-specific options and credentials.", + "properties": { + "gmsaCredentialSpec": { + "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.", + "type": "string" + }, + "gmsaCredentialSpecName": { + "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.", + "type": "string" + } + }, "type": "object" }, "io.k8s.api.events.v1beta1.Event": { @@ -16606,7 +16616,7 @@ "description": "CustomResourceSubresourceScale defines how to serve the scale subresource for CustomResources.", "properties": { "labelSelectorPath": { - "description": "LabelSelectorPath defines the JSON path inside of a CustomResource that corresponds to Scale.Status.Selector. Only JSON paths without the array notation are allowed. Must be a JSON Path under .status. Must be set to work with HPA. If there is no value under the given path in the CustomResource, the status label selector value in the /scale subresource will default to the empty string.", + "description": "LabelSelectorPath defines the JSON path inside of a CustomResource that corresponds to Scale.Status.Selector. Only JSON paths without the array notation are allowed. Must be a JSON Path under .status or .spec. Must be set to work with HPA. The field pointed by this JSON path must be a string field (not a complex selector struct) which contains a serialized label selector in string form. More info: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions#scale-subresource If there is no value under the given path in the CustomResource, the status label selector value in the /scale subresource will default to the empty string.", "type": "string" }, "specReplicasPath": { diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index 7ebac78fe28..4ff89886741 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -78,7 +78,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: heapster-nanny resources: limits: @@ -114,7 +114,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index e21f18c3445..9f359f41418 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -79,7 +79,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: heapster-nanny resources: limits: @@ -115,7 +115,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 292a62dd228..c5b78d12680 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -78,7 +78,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=influxdb:http://monitoring-influxdb:8086 - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: heapster-nanny resources: limits: @@ -114,7 +114,7 @@ spec: # Specifies the smallest cluster (defined in number of nodes) # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: eventer-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index cbe891e929e..6bb898baf91 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -81,7 +81,7 @@ spec: fieldRef: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: heapster-nanny resources: limits: diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index a0f79fea0af..414bb1af00c 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -59,7 +59,7 @@ spec: command: - /heapster - --source=kubernetes.summary_api:'' - - image: k8s.gcr.io/addon-resizer:1.8.4 + - image: k8s.gcr.io/addon-resizer:1.8.5 name: heapster-nanny resources: limits: diff --git a/cluster/addons/metrics-server/metrics-server-deployment.yaml b/cluster/addons/metrics-server/metrics-server-deployment.yaml index 4dccf59b34f..2306dc98f9f 100644 --- a/cluster/addons/metrics-server/metrics-server-deployment.yaml +++ b/cluster/addons/metrics-server/metrics-server-deployment.yaml @@ -63,7 +63,7 @@ spec: name: https protocol: TCP - name: metrics-server-nanny - image: k8s.gcr.io/addon-resizer:1.8.4 + image: k8s.gcr.io/addon-resizer:1.8.5 resources: limits: cpu: 100m diff --git a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml index b12bedf5807..5d81e8e002d 100644 --- a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml +++ b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml @@ -39,7 +39,7 @@ spec: initialDelaySeconds: 5 timeoutSeconds: 5 - name: addon-resizer - image: k8s.gcr.io/addon-resizer:1.8.4 + image: k8s.gcr.io/addon-resizer:1.8.5 resources: limits: cpu: 100m diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 390155f9761..81ffb0bb097 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -523,3 +523,5 @@ WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=win1809:NoSche # Whether to set up a private GCE cluster, i.e. a cluster where nodes have only private IPs. GCE_PRIVATE_CLUSTER="${KUBE_GCE_PRIVATE_CLUSTER:-false}" + +ETCD_LISTEN_CLIENT_IP=0.0.0.0 diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 8208859b6d6..37190119ef4 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -611,6 +611,15 @@ function create-master-auth { if [[ -n "${ADDON_MANAGER_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${ADDON_MANAGER_TOKEN}," "system:addon-manager,uid:system:addon-manager,system:masters" fi + if [[ -n "${EXTRA_STATIC_AUTH_COMPONENTS:-}" ]]; then + # Create a static Bearer token and kubeconfig for extra, comma-separated components. + IFS="," read -r -a extra_components <<< "${EXTRA_STATIC_AUTH_COMPONENTS:-}" + for extra_component in "${extra_components[@]}"; do + local token="$(secure_random 32)" + append_or_replace_prefixed_line "${known_tokens_csv}" "${token}," "system:${extra_component},uid:system:${extra_component}" + create-kubeconfig "${extra_component}" "${token}" + done + fi local use_cloud_config="false" cat </etc/gce.conf [global] @@ -1405,6 +1414,7 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *host_ip *}}@$host_ip@g" "${temp_file}" sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}" + sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}" # Get default storage backend from manifest file. local -r default_storage_backend=$(cat "${temp_file}" | \ grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \ diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest index 4500d1df9e6..fa54fbc0725 100644 --- a/cluster/gce/manifests/etcd.manifest +++ b/cluster/gce/manifests/etcd.manifest @@ -23,7 +23,7 @@ "command": [ "/bin/sh", "-c", - "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], "env": [ { "name": "TARGET_STORAGE", diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index d9facf9aab8..775745bd0e5 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -730,7 +730,6 @@ function construct-common-kubelet-flags { function construct-linux-kubelet-flags { local master="$1" local flags="$(construct-common-kubelet-flags)" - flags+=" --allow-privileged=true" # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" flags+=" --experimental-check-node-capabilities-before-mount=true" @@ -1406,6 +1405,11 @@ EOF if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then cat >>$file <>$file < 0 { + return errors.New("ignorePreflightErrors field is not supported by v1beta1 config format") + } + + return nil +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go index 0e7b73bbd73..0609c31e0a7 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go @@ -37,6 +37,14 @@ func TestInternalToVersionedInitConfigurationConversion(t *testing.T) { }, expectedError: true, }, + "ignorePreflightErrors set causes an error": { + in: kubeadm.InitConfiguration{ + NodeRegistration: kubeadm.NodeRegistrationOptions{ + IgnorePreflightErrors: []string{"SomeUndesirableError"}, + }, + }, + expectedError: true, + }, } for name, tc := range testcases { t.Run(name, func(t *testing.T) { @@ -51,6 +59,66 @@ func TestInternalToVersionedInitConfigurationConversion(t *testing.T) { } } +func TestInternalToVersionedJoinConfigurationConversion(t *testing.T) { + testcases := map[string]struct { + in kubeadm.JoinConfiguration + expectedError bool + }{ + "conversion succeeds": { + in: kubeadm.JoinConfiguration{}, + expectedError: false, + }, + "ignorePreflightErrors set causes an error": { + in: kubeadm.JoinConfiguration{ + NodeRegistration: kubeadm.NodeRegistrationOptions{ + IgnorePreflightErrors: []string{"SomeUndesirableError"}, + }, + }, + expectedError: true, + }, + } + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + versioned := &JoinConfiguration{} + err := Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(&tc.in, versioned, nil) + if err == nil && tc.expectedError { + t.Error("unexpected success") + } else if err != nil && !tc.expectedError { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestInternalToVersionedNodeRegistrationOptionsConversion(t *testing.T) { + testcases := map[string]struct { + in kubeadm.NodeRegistrationOptions + expectedError bool + }{ + "conversion succeeds": { + in: kubeadm.NodeRegistrationOptions{}, + expectedError: false, + }, + "ignorePreflightErrors set causes an error": { + in: kubeadm.NodeRegistrationOptions{ + IgnorePreflightErrors: []string{"SomeUndesirableError"}, + }, + expectedError: true, + }, + } + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + versioned := &NodeRegistrationOptions{} + err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&tc.in, versioned, nil) + if err == nil && tc.expectedError { + t.Error("unexpected success") + } else if err != nil && !tc.expectedError { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + func TestInternalToVersionedJoinControlPlaneConversion(t *testing.T) { testcases := map[string]struct { in kubeadm.JoinControlPlane diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go index 85231880198..3e2ab2bed5d 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go @@ -22,5 +22,5 @@ const ( // DefaultCACertPath defines default location of CA certificate on Windows DefaultCACertPath = "C:/etc/kubernetes/pki/ca.crt" // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "tcp" + DefaultUrlScheme = "npipe" ) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go index 4b2d3007945..012e3a71b63 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go @@ -257,6 +257,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*kubeadm.NodeRegistrationOptions)(nil), (*NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(a.(*kubeadm.NodeRegistrationOptions), b.(*NodeRegistrationOptions), scope) + }); err != nil { + return err + } return nil } @@ -848,10 +853,6 @@ func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOpti out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + // WARNING: in.IgnorePreflightErrors requires manual conversion: does not exist in peer-type return nil } - -// Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in, out, s) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go index 7d4d3b82cb0..9f0986db46e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go @@ -22,5 +22,5 @@ const ( // DefaultCACertPath defines default location of CA certificate on Windows DefaultCACertPath = "C:/etc/kubernetes/pki/ca.crt" // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "tcp" + DefaultUrlScheme = "npipe" ) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go index 327463430c6..d9d578b8108 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go @@ -215,6 +215,9 @@ type NodeRegistrationOptions struct { // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` + + // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered. + IgnorePreflightErrors []string `json:"ignorePreflightErrors,omitempty"` } // Networking contains elements describing cluster's networking configuration diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go index 368d760a47c..d7498eb1fa1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go @@ -821,6 +821,7 @@ func autoConvert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOpti out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) return nil } @@ -834,6 +835,7 @@ func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOpti out.CRISocket = in.CRISocket out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) + out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go index 4f611d9a281..6e4945ce374 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go @@ -538,6 +538,11 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { (*out)[key] = val } } + if in.IgnorePreflightErrors != nil { + in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD index 84c37d0ab40..076ebdc9913 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD @@ -34,6 +34,7 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", "//pkg/proxy/apis/config:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index d61b4603377..37f62d026a0 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -463,12 +463,25 @@ func ValidateAPIEndpoint(c *kubeadm.APIEndpoint, fldPath *field.Path) field.Erro return allErrs } -// ValidateIgnorePreflightErrors validates duplicates in ignore-preflight-errors flag. -func ValidateIgnorePreflightErrors(ignorePreflightErrors []string) (sets.String, error) { +// ValidateIgnorePreflightErrors validates duplicates in: +// - ignore-preflight-errors flag and +// - ignorePreflightErrors field in {Init,Join}Configuration files. +func ValidateIgnorePreflightErrors(ignorePreflightErrorsFromCLI, ignorePreflightErrorsFromConfigFile []string) (sets.String, error) { ignoreErrors := sets.NewString() allErrs := field.ErrorList{} - for _, item := range ignorePreflightErrors { + for _, item := range ignorePreflightErrorsFromConfigFile { + ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive + } + + if ignoreErrors.Has("all") { + // "all" is forbidden in config files. Administrators should use an + // explicit list of errors they want to ignore, as it can be risky to + // mask all errors in such a way. Hence, we return an error: + allErrs = append(allErrs, field.Invalid(field.NewPath("ignorePreflightErrors"), "all", "'all' cannot be used in configuration file")) + } + + for _, item := range ignorePreflightErrorsFromCLI { ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive } diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index 05be00ab90b..3dba33f0224 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" @@ -703,26 +704,81 @@ func TestValidateFeatureGates(t *testing.T) { func TestValidateIgnorePreflightErrors(t *testing.T) { var tests = []struct { - ignorePreflightErrors []string - expectedLen int - expectedError bool + ignorePreflightErrorsFromCLI []string + ignorePreflightErrorsFromConfigFile []string + expectedSet sets.String + expectedError bool }{ - {[]string{}, 0, false}, // empty list - {[]string{"check1", "check2"}, 2, false}, // non-duplicate - {[]string{"check1", "check2", "check1"}, 2, false}, // duplicates - {[]string{"check1", "check2", "all"}, 3, true}, // non-duplicate, but 'all' present together wth individual checks - {[]string{"all"}, 1, false}, // skip all checks by using new flag - {[]string{"all"}, 1, false}, // skip all checks by using both old and new flags at the same time + { // empty lists in CLI and config file + []string{}, + []string{}, + sets.NewString(), + false, + }, + { // empty list in CLI only + []string{}, + []string{"a"}, + sets.NewString("a"), + false, + }, + { // empty list in config file only + []string{"a"}, + []string{}, + sets.NewString("a"), + false, + }, + { // no duplicates, no overlap + []string{"a", "b"}, + []string{"c", "d"}, + sets.NewString("a", "b", "c", "d"), + false, + }, + { // some duplicates, with some overlapping duplicates + []string{"a", "b", "a"}, + []string{"c", "b"}, + sets.NewString("a", "b", "c"), + false, + }, + { // non-duplicate, but 'all' present together with individual checks in CLI + []string{"a", "b", "all"}, + []string{}, + sets.NewString(), + true, + }, + { // empty list in CLI, but 'all' present in config file, which is forbidden + []string{}, + []string{"all"}, + sets.NewString(), + true, + }, + { // non-duplicate, but 'all' present in config file, which is forbidden + []string{"a", "b"}, + []string{"all"}, + sets.NewString(), + true, + }, + { // non-duplicate, but 'all' present in CLI, while values are in config file, which is forbidden + []string{"all"}, + []string{"a", "b"}, + sets.NewString(), + true, + }, + { // skip all checks + []string{"all"}, + []string{}, + sets.NewString("all"), + false, + }, } for _, rt := range tests { - result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrors) + result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile) switch { case err != nil && !rt.expectedError: - t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s), error: %v", rt.ignorePreflightErrors, err) + t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s, %s), error: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, err) case err == nil && rt.expectedError: - t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s) but got: %v", rt.ignorePreflightErrors, result) - case result.Len() != rt.expectedLen: - t.Errorf("ValidateIgnorePreflightErrors: expected Len = %d for input (%s) but got: %v, %v", rt.expectedLen, rt.ignorePreflightErrors, result.Len(), result) + t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s, %s) but got: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result) + case err == nil && !result.Equal(rt.expectedSet): + t.Errorf("ValidateIgnorePreflightErrors: expected (%v) for input (%s, %s) but got: %v", rt.expectedSet, rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result) } } } diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index ebdf10bb2e2..164010242b0 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -567,6 +567,11 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { (*out)[key] = val } } + if in.IgnorePreflightErrors != nil { + in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index dcaed8131f3..45ab52a9dfc 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -99,6 +99,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 9f4e3f1fdfc..8eb7467d45f 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -103,7 +103,7 @@ type initOptions struct { // compile-time assert that the local data object satisfies the phases data interface. var _ phases.InitData = &initData{} -// initData defines all the runtime information used when running the kubeadm init worklow; +// initData defines all the runtime information used when running the kubeadm init workflow; // this data is shared across all the phases that are included in the workflow. type initData struct { cfg *kubeadmapi.InitConfiguration @@ -296,11 +296,6 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io return nil, err } - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors) - if err != nil { - return nil, err - } - if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil { return nil, err } @@ -316,6 +311,13 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io return nil, err } + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, err + } + // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + // override node name and CRI socket from the command line options if options.externalcfg.NodeRegistration.Name != "" { cfg.NodeRegistration.Name = options.externalcfg.NodeRegistration.Name diff --git a/cmd/kubeadm/app/cmd/init_test.go b/cmd/kubeadm/app/cmd/init_test.go index 79855fbca14..2daed1f77c1 100644 --- a/cmd/kubeadm/app/cmd/init_test.go +++ b/cmd/kubeadm/app/cmd/init_test.go @@ -23,6 +23,7 @@ import ( "path/filepath" "testing" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/features" ) @@ -38,6 +39,9 @@ bootstrapTokens: nodeRegistration: criSocket: /run/containerd/containerd.sock name: someName + ignorePreflightErrors: + - c + - d --- apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration @@ -129,6 +133,30 @@ func TestNewInitData(t *testing.T) { }, expectError: true, }, + + // Pre-flight errors: + { + name: "pre-flights errors from CLI args only", + flags: map[string]string{ + options.IgnorePreflightErrors: "a,b", + }, + validate: expectedInitIgnorePreflightErrors("a", "b"), + }, + { + name: "pre-flights errors from InitConfiguration only", + flags: map[string]string{ + options.CfgPath: configFilePath, + }, + validate: expectedInitIgnorePreflightErrors("c", "d"), + }, + { + name: "pre-flights errors from both CLI args and InitConfiguration", + flags: map[string]string{ + options.CfgPath: configFilePath, + options.IgnorePreflightErrors: "a,b", + }, + validate: expectedInitIgnorePreflightErrors("a", "b", "c", "d"), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -157,3 +185,15 @@ func TestNewInitData(t *testing.T) { }) } } + +func expectedInitIgnorePreflightErrors(expectedItems ...string) func(t *testing.T, data *initData) { + expected := sets.NewString(expectedItems...) + return func(t *testing.T, data *initData) { + if !expected.Equal(data.ignorePreflightErrors) { + t.Errorf("Invalid ignore preflight errors. Expected: %v. Actual: %v", expected.List(), data.ignorePreflightErrors.List()) + } + if !expected.HasAll(data.cfg.NodeRegistration.IgnorePreflightErrors...) { + t.Errorf("Invalid ignore preflight errors in InitConfiguration. Expected: %v. Actual: %v", expected.List(), data.cfg.NodeRegistration.IgnorePreflightErrors) + } + } +} diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 0266ea46327..a6ee510892d 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -133,7 +133,7 @@ type joinOptions struct { // compile-time assert that the local data object satisfies the phases data interface. var _ phases.JoinData = &joinData{} -// joinData defines all the runtime information used when running the kubeadm join worklow; +// joinData defines all the runtime information used when running the kubeadm join workflow; // this data is shared across all the phases that are included in the workflow. type joinData struct { cfg *kubeadmapi.JoinConfiguration @@ -349,12 +349,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri } } - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors) - if err != nil { - return nil, err - } - - if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil { + if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { return nil, err } @@ -383,6 +378,13 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri return nil, err } + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, err + } + // Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + // override node name and CRI socket from the command line opt if opt.externalcfg.NodeRegistration.Name != "" { cfg.NodeRegistration.Name = opt.externalcfg.NodeRegistration.Name diff --git a/cmd/kubeadm/app/cmd/join_test.go b/cmd/kubeadm/app/cmd/join_test.go index 211fbcd2918..8a466a9f68b 100644 --- a/cmd/kubeadm/app/cmd/join_test.go +++ b/cmd/kubeadm/app/cmd/join_test.go @@ -22,6 +22,7 @@ import ( "path/filepath" "testing" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" ) @@ -36,6 +37,9 @@ discovery: nodeRegistration: criSocket: /run/containerd/containerd.sock name: someName + ignorePreflightErrors: + - c + - d ` ) @@ -215,6 +219,31 @@ func TestNewJoinData(t *testing.T) { }, expectError: true, }, + + // Pre-flight errors: + { + name: "pre-flights errors from CLI args only", + flags: map[string]string{ + options.IgnorePreflightErrors: "a,b", + options.FileDiscovery: "https://foo", //required only to pass discovery validation + }, + validate: expectedJoinIgnorePreflightErrors(sets.NewString("a", "b")), + }, + { + name: "pre-flights errors from JoinConfiguration only", + flags: map[string]string{ + options.CfgPath: configFilePath, + }, + validate: expectedJoinIgnorePreflightErrors(sets.NewString("c", "d")), + }, + { + name: "pre-flights errors from both CLI args and JoinConfiguration", + flags: map[string]string{ + options.CfgPath: configFilePath, + options.IgnorePreflightErrors: "a,b", + }, + validate: expectedJoinIgnorePreflightErrors(sets.NewString("a", "b", "c", "d")), + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -243,3 +272,14 @@ func TestNewJoinData(t *testing.T) { }) } } + +func expectedJoinIgnorePreflightErrors(expected sets.String) func(t *testing.T, data *joinData) { + return func(t *testing.T, data *joinData) { + if !expected.Equal(data.ignorePreflightErrors) { + t.Errorf("Invalid ignore preflight errors. Expected: %v. Actual: %v", expected.List(), data.ignorePreflightErrors.List()) + } + if !expected.HasAll(data.cfg.NodeRegistration.IgnorePreflightErrors...) { + t.Errorf("Invalid ignore preflight errors in JoinConfiguration. Expected: %v. Actual: %v", expected.List(), data.cfg.NodeRegistration.IgnorePreflightErrors) + } + } +} diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 1ef20d7ca3c..52d7a989096 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -59,7 +59,7 @@ type resetOptions struct { kubeconfigPath string } -// resetData defines all the runtime information used when running the kubeadm reset worklow; +// resetData defines all the runtime information used when running the kubeadm reset workflow; // this data is shared across all the phases that are included in the workflow. type resetData struct { certificatesDir string @@ -84,10 +84,6 @@ func newResetOptions() *resetOptions { // newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow. func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) { var cfg *kubeadmapi.InitConfiguration - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors) - if err != nil { - return nil, err - } client, err := getClientset(options.kubeconfigPath, false) if err == nil { @@ -100,6 +96,16 @@ func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out i klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath) } + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrors(cfg)) + if err != nil { + return nil, err + } + kubeadmutil.CheckErr(err) + if cfg != nil { + // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + } + var criSocketPath string if options.criSocketPath == "" { criSocketPath, err = resetDetectCRISocket(cfg) @@ -121,6 +127,13 @@ func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out i }, nil } +func ignorePreflightErrors(cfg *kubeadmapi.InitConfiguration) []string { + if cfg == nil { + return []string{} + } + return cfg.NodeRegistration.IgnorePreflightErrors +} + // AddResetFlags adds reset flags func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) { flagSet.StringVar( diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 6c384337b64..c47115f6197 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -76,17 +76,6 @@ func getK8sVersionFromUserInput(flags *applyPlanFlags, args []string, versionIsM // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion string) (clientset.Interface, upgrade.VersionGetter, *kubeadmapi.InitConfiguration, error) { - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(flags.ignorePreflightErrors) - if err != nil { - return nil, nil, nil, err - } - - // Ensure the user is root - klog.V(1).Info("running preflight checks") - if err := runPreflightChecks(ignorePreflightErrorsSet); err != nil { - return nil, nil, nil, err - } - client, err := getClient(flags.kubeConfigPath, dryRun) if err != nil { return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) @@ -97,11 +86,6 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin return nil, nil, nil, errors.New("cannot upgrade a self-hosted control plane") } - // Run healthchecks against the cluster - if err := upgrade.CheckClusterHealth(client, ignorePreflightErrorsSet); err != nil { - return nil, nil, nil, errors.Wrap(err, "[upgrade/health] FATAL") - } - // Fetch the configuration from a file or ConfigMap and validate it fmt.Println("[upgrade/config] Making sure the configuration is correct:") @@ -112,6 +96,24 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin cfg, err = configutil.FetchInitConfigurationFromCluster(client, os.Stdout, "upgrade/config", false) } + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(flags.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) + if err != nil { + return nil, nil, nil, err + } + // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: + cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() + + // Ensure the user is root + klog.V(1).Info("running preflight checks") + if err := runPreflightChecks(ignorePreflightErrorsSet); err != nil { + return nil, nil, nil, err + } + + // Run healthchecks against the cluster + if err := upgrade.CheckClusterHealth(client, ignorePreflightErrorsSet); err != nil { + return nil, nil, nil, errors.Wrap(err, "[upgrade/health] FATAL") + } + if err != nil { if apierrors.IsNotFound(err) { fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) diff --git a/cmd/kubeadm/app/constants/constants_windows.go b/cmd/kubeadm/app/constants/constants_windows.go index 4ee63a3ef4a..6daae0a1fff 100644 --- a/cmd/kubeadm/app/constants/constants_windows.go +++ b/cmd/kubeadm/app/constants/constants_windows.go @@ -20,5 +20,5 @@ package constants const ( // DefaultDockerCRISocket defines the default Docker CRI socket - DefaultDockerCRISocket = "tcp://localhost:2375" + DefaultDockerCRISocket = "npipe:////./pipe/docker_engine" ) diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml index f42d1cce4e4..e0af329be05 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml @@ -15,6 +15,7 @@ Discovery: Timeout: 5m0s NodeRegistration: CRISocket: /var/run/dockershim.sock + IgnorePreflightErrors: null KubeletExtraArgs: null Name: control-plane-1 Taints: diff --git a/cmd/kubeadm/app/util/runtime/BUILD b/cmd/kubeadm/app/util/runtime/BUILD index 27e839bb8d5..e320f461cf0 100644 --- a/cmd/kubeadm/app/util/runtime/BUILD +++ b/cmd/kubeadm/app/util/runtime/BUILD @@ -2,7 +2,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["runtime.go"], + srcs = [ + "runtime.go", + "runtime_unix.go", + "runtime_windows.go", + ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime", visibility = ["//visibility:public"], deps = [ @@ -10,7 +14,12 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/github.com/Microsoft/go-winio:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go index bd4a168db8f..71e2c04521a 100644 --- a/cmd/kubeadm/app/util/runtime/runtime.go +++ b/cmd/kubeadm/app/util/runtime/runtime.go @@ -17,7 +17,6 @@ limitations under the License. package util import ( - "os" "path/filepath" goruntime "runtime" "strings" @@ -180,23 +179,8 @@ func (runtime *DockerRuntime) ImageExists(image string) (bool, error) { return err == nil, nil } -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - - return fileInfo.Mode()&os.ModeSocket != 0 -} - // detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead func detectCRISocketImpl(isSocket func(string) bool) (string, error) { - const ( - dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible - containerdSocket = "/run/containerd/containerd.sock" - ) - foundCRISockets := []string{} knownCRISockets := []string{ // Docker and containerd sockets are special cased below, hence not to be included here @@ -233,9 +217,5 @@ func detectCRISocketImpl(isSocket func(string) bool) (string, error) { // DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. func DetectCRISocket() (string, error) { - if goruntime.GOOS != "linux" { - return constants.DefaultDockerCRISocket, nil - } - return detectCRISocketImpl(isExistingSocket) } diff --git a/cmd/kubeadm/app/util/runtime/runtime_unix.go b/cmd/kubeadm/app/util/runtime/runtime_unix.go new file mode 100644 index 00000000000..b15c3037313 --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/runtime_unix.go @@ -0,0 +1,38 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "os" +) + +const ( + dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible + containerdSocket = "/run/containerd/containerd.sock" +) + +// isExistingSocket checks if path exists and is domain socket +func isExistingSocket(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + + return fileInfo.Mode()&os.ModeSocket != 0 +} diff --git a/cmd/kubeadm/app/util/runtime/runtime_windows.go b/cmd/kubeadm/app/util/runtime/runtime_windows.go new file mode 100644 index 00000000000..0c6a7b496dc --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/runtime_windows.go @@ -0,0 +1,38 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + winio "github.com/Microsoft/go-winio" +) + +const ( + dockerSocket = "//./pipe/docker_engine" // The Docker socket is not CRI compatible + containerdSocket = "//./pipe/containerd-containerd" // Proposed containerd named pipe for Windows +) + +// isExistingSocket checks if path exists and is domain socket +func isExistingSocket(path string) bool { + _, err := winio.DialPipe(path, nil) + if err != nil { + return false + } + + return true +} diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index a8b13f757ff..3c54efc5f16 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -135,6 +135,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library", + "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 213999036a4..66b054100a5 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -39,7 +39,6 @@ import ( kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" "k8s.io/kubernetes/pkg/kubelet/config" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/master/ports" utilflag "k8s.io/kubernetes/pkg/util/flag" utiltaints "k8s.io/kubernetes/pkg/util/taints" @@ -195,19 +194,6 @@ type KubeletFlags struct { // This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node. // This can be useful for debugging volume related issues. KeepTerminatedPodVolumes bool - // allowPrivileged enables containers to request privileged mode. - // Defaults to true. - AllowPrivileged bool - // hostNetworkSources is a comma-separated list of sources from which the - // Kubelet allows pods to use of host network. Defaults to "*". Valid - // options are "file", "http", "api", and "*" (all sources). - HostNetworkSources []string - // hostPIDSources is a comma-separated list of sources from which the - // Kubelet allows pods to use the host pid namespace. Defaults to "*". - HostPIDSources []string - // hostIPCSources is a comma-separated list of sources from which the - // Kubelet allows pods to use the host ipc namespace. Defaults to "*". - HostIPCSources []string } // NewKubeletFlags will create a new KubeletFlags with default values @@ -236,11 +222,6 @@ func NewKubeletFlags() *KubeletFlags { VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", RegisterNode: true, SeccompProfileRoot: filepath.Join(defaultRootDir, "seccomp"), - HostNetworkSources: []string{kubetypes.AllSource}, - HostPIDSources: []string{kubetypes.AllSource}, - HostIPCSources: []string{kubetypes.AllSource}, - // TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated - AllowPrivileged: true, // prior to the introduction of this flag, there was a hardcoded cap of 50 images NodeStatusMaxImages: 50, } @@ -443,18 +424,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.MarkDeprecated("non-masquerade-cidr", "will be removed in a future version") fs.BoolVar(&f.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", f.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates. Can be useful for debugging volume related issues.") fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version") - // TODO(#58010:v1.13.0): Remove --allow-privileged, it is deprecated - fs.BoolVar(&f.AllowPrivileged, "allow-privileged", f.AllowPrivileged, "If true, allow containers to request privileged mode. Default: true") - fs.MarkDeprecated("allow-privileged", "will be removed in a future version") - // TODO(#58010:v1.12.0): Remove --host-network-sources, it is deprecated - fs.StringSliceVar(&f.HostNetworkSources, "host-network-sources", f.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network.") - fs.MarkDeprecated("host-network-sources", "will be removed in a future version") - // TODO(#58010:v1.12.0): Remove --host-pid-sources, it is deprecated - fs.StringSliceVar(&f.HostPIDSources, "host-pid-sources", f.HostPIDSources, "Comma-separated list of sources from which the Kubelet allows pods to use the host pid namespace.") - fs.MarkDeprecated("host-pid-sources", "will be removed in a future version") - // TODO(#58010:v1.12.0): Remove --host-ipc-sources, it is deprecated - fs.StringSliceVar(&f.HostIPCSources, "host-ipc-sources", f.HostIPCSources, "Comma-separated list of sources from which the Kubelet allows pods to use the host ipc namespace.") - fs.MarkDeprecated("host-ipc-sources", "will be removed in a future version") } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 4ce3e470373..d9e2d0525eb 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -57,6 +57,7 @@ import ( "k8s.io/client-go/tools/record" certutil "k8s.io/client-go/util/cert" "k8s.io/client-go/util/certificate" + "k8s.io/client-go/util/connrotation" "k8s.io/client-go/util/keyutil" cloudprovider "k8s.io/cloud-provider" cliflag "k8s.io/component-base/cli/flag" @@ -567,6 +568,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan if err != nil { return err } + if closeAllConns == nil { + return errors.New("closeAllConns must be a valid function other than nil") + } kubeDeps.OnHeartbeatFailure = closeAllConns kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) @@ -806,8 +810,21 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) } kubeClientConfigOverrides(s, clientConfig) + closeAllConns, err := updateDialer(clientConfig) + if err != nil { + return nil, nil, err + } + return clientConfig, closeAllConns, nil +} - return clientConfig, nil, nil +// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections. +func updateDialer(clientConfig *restclient.Config) (func(), error) { + if clientConfig.Transport != nil || clientConfig.Dial != nil { + return nil, fmt.Errorf("there is already a transport or dialer configured") + } + d := connrotation.NewDialer((&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext) + clientConfig.Dial = d.DialContext + return d.CloseAll, nil } // buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate @@ -974,32 +991,9 @@ func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencie // Setup event recorder if required. makeEventRecorder(kubeDeps, nodeName) - // TODO(mtaufen): I moved the validation of these fields here, from UnsecuredKubeletConfig, - // so that I could remove the associated fields from KubeletConfiginternal. I would - // prefer this to be done as part of an independent validation step on the - // KubeletConfiguration. But as far as I can tell, we don't have an explicit - // place for validation of the KubeletConfiguration yet. - hostNetworkSources, err := kubetypes.GetValidatedSources(kubeServer.HostNetworkSources) - if err != nil { - return err - } - - hostPIDSources, err := kubetypes.GetValidatedSources(kubeServer.HostPIDSources) - if err != nil { - return err - } - - hostIPCSources, err := kubetypes.GetValidatedSources(kubeServer.HostIPCSources) - if err != nil { - return err - } - - privilegedSources := capabilities.PrivilegedSources{ - HostNetworkSources: hostNetworkSources, - HostPIDSources: hostPIDSources, - HostIPCSources: hostIPCSources, - } - capabilities.Setup(kubeServer.AllowPrivileged, privilegedSources, 0) + capabilities.Initialize(capabilities.Capabilities{ + AllowPrivileged: true, + }) credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory) klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) diff --git a/go.mod b/go.mod index 747e8871346..f48c41828ad 100644 --- a/go.mod +++ b/go.mod @@ -178,7 +178,7 @@ require ( k8s.io/csi-translation-lib v0.0.0 k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af k8s.io/heapster v1.2.0-beta.1 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/kube-aggregator v0.0.0 k8s.io/kube-controller-manager v0.0.0 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 @@ -442,7 +442,7 @@ replace ( k8s.io/csi-translation-lib => ./staging/src/k8s.io/csi-translation-lib k8s.io/gengo => k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af k8s.io/heapster => k8s.io/heapster v1.2.0-beta.1 - k8s.io/klog => k8s.io/klog v0.3.0 + k8s.io/klog => k8s.io/klog v0.3.1 k8s.io/kube-aggregator => ./staging/src/k8s.io/kube-aggregator k8s.io/kube-controller-manager => ./staging/src/k8s.io/kube-controller-manager k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 diff --git a/go.sum b/go.sum index 9f14ece4d94..fba2f8cf341 100644 --- a/go.sum +++ b/go.sum @@ -462,8 +462,8 @@ k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8 k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/heapster v1.2.0-beta.1 h1:lUsE/AHOMHpi3MLlBEkaU8Esxm5QhdyCrv1o7ot0s84= k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3 h1:WD6cPA3q7qxZe6Fwir0XjjGwGMaWbHlHUcjCcOzuRG0= diff --git a/hack/.golint_failures b/hack/.golint_failures index 46a58598f9c..c2be7af5e05 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -537,7 +537,6 @@ staging/src/k8s.io/client-go/scale/scheme/autoscalingv1 staging/src/k8s.io/client-go/scale/scheme/extensionsv1beta1 staging/src/k8s.io/client-go/scale/scheme/extensionsv1beta1 staging/src/k8s.io/client-go/testing -staging/src/k8s.io/client-go/tools/auth staging/src/k8s.io/client-go/tools/cache staging/src/k8s.io/client-go/tools/cache/testing staging/src/k8s.io/client-go/tools/clientcmd @@ -546,7 +545,6 @@ staging/src/k8s.io/client-go/tools/clientcmd/api/latest staging/src/k8s.io/client-go/tools/clientcmd/api/v1 staging/src/k8s.io/client-go/tools/leaderelection staging/src/k8s.io/client-go/tools/leaderelection/resourcelock -staging/src/k8s.io/client-go/tools/portforward staging/src/k8s.io/client-go/tools/record staging/src/k8s.io/client-go/tools/reference staging/src/k8s.io/client-go/transport @@ -590,12 +588,10 @@ staging/src/k8s.io/sample-apiserver/pkg/apis/wardle staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1 staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/fischer staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/flunder -test/e2e/autoscaling test/e2e/chaosmonkey test/e2e/common test/e2e/lifecycle/bootstrap test/e2e/scalability -test/e2e/storage/drivers test/e2e/storage/testsuites test/e2e/storage/vsphere test/e2e_kubeadm diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index ba084ea3f72..237357112da 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -65,6 +65,7 @@ ENABLE_CLUSTER_DNS=${KUBE_ENABLE_CLUSTER_DNS:-true} ENABLE_NODELOCAL_DNS=${KUBE_ENABLE_NODELOCAL_DNS:-false} DNS_SERVER_IP=${KUBE_DNS_SERVER_IP:-10.0.0.10} LOCAL_DNS_IP=${KUBE_LOCAL_DNS_IP:-169.254.20.10} +DNS_MEMORY_LIMIT=${KUBE_DNS_MEMORY_LIMIT:-170Mi} DNS_DOMAIN=${KUBE_DNS_NAME:-"cluster.local"} KUBECTL=${KUBECTL:-"${KUBE_ROOT}/cluster/kubectl.sh"} WAIT_FOR_URL_API_SERVER=${WAIT_FOR_URL_API_SERVER:-60} @@ -686,11 +687,6 @@ function start_kubelet { KUBELET_LOG=${LOG_DIR}/kubelet.log mkdir -p "${POD_MANIFEST_PATH}" &>/dev/null || sudo mkdir -p "${POD_MANIFEST_PATH}" - priv_arg="" - if [[ -n "${ALLOW_PRIVILEGED}" ]]; then - priv_arg="--allow-privileged=${ALLOW_PRIVILEGED}" - fi - cloud_config_arg=("--cloud-provider=${CLOUD_PROVIDER}" "--cloud-config=${CLOUD_CONFIG}") if [[ "${EXTERNAL_CLOUD_PROVIDER:-}" == "true" ]]; then cloud_config_arg=("--cloud-provider=external") @@ -751,7 +747,6 @@ function start_kubelet { # shellcheck disable=SC2206 all_kubelet_flags=( - ${priv_arg:+"$priv_arg"} "--v=${LOG_LEVEL}" "--vmodule=${LOG_SPEC}" "--chaos-chance=${CHAOS_CHANCE}" @@ -849,6 +844,7 @@ function start_kubedns { cp "${KUBE_ROOT}/cluster/addons/dns/kube-dns/kube-dns.yaml.in" kube-dns.yaml ${SED} -i -e "s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g" kube-dns.yaml ${SED} -i -e "s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" kube-dns.yaml + ${SED} -i -e "s/{{ pillar\['dns_memory_limit'\] }}/${DNS_MEMORY_LIMIT}/g" kube-dns.yaml # TODO update to dns role once we have one. # use kubectl to create kubedns addon ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" --namespace=kube-system create -f kube-dns.yaml diff --git a/hack/pin-dependency.sh b/hack/pin-dependency.sh index c5c3fc1d794..ee14cf6db26 100755 --- a/hack/pin-dependency.sh +++ b/hack/pin-dependency.sh @@ -29,8 +29,6 @@ source "${KUBE_ROOT}/hack/lib/init.sh" # Explicitly opt into go modules, even though we're inside a GOPATH directory export GO111MODULE=on -# Explicitly clear GOPATH, to ensure nothing this script calls makes use of that path info -export GOPATH= # Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor export GOFLAGS= # Detect problematic GOPROXY settings that prevent lookup of dependencies diff --git a/hack/verify-symbols.sh b/hack/verify-symbols.sh index 8f0ef8dce3d..77069cfd799 100755 --- a/hack/verify-symbols.sh +++ b/hack/verify-symbols.sh @@ -23,21 +23,36 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env -make -C "${KUBE_ROOT}" WHAT=cmd/hyperkube +kube::util::ensure-temp-dir +OUTPUT="${KUBE_TEMP}"/symbols-output +cleanup() { + rm -rf "${OUTPUT}" +} +trap "cleanup" EXIT SIGINT +mkdir -p "${OUTPUT}" + +GOLDFLAGS="-w" make -C "${KUBE_ROOT}" WHAT=cmd/hyperkube # Add other BADSYMBOLS here. BADSYMBOLS=( "httptest" "testify" "testing[.]" + "TestOnlySetFatalOnDecodeError" + "TrackStorageCleanup" ) # b/c hyperkube binds everything simply check that for bad symbols -SYMBOLS="$(nm "${KUBE_OUTPUT_HOSTBIN}/hyperkube")" +go tool nm "${KUBE_OUTPUT_HOSTBIN}/hyperkube" > "${OUTPUT}/hyperkube-symbols" + +if ! grep -q "NewHyperKubeCommand" "${OUTPUT}/hyperkube-symbols"; then + echo "No symbols found in hyperkube binary." + exit 1 +fi RESULT=0 for BADSYMBOL in "${BADSYMBOLS[@]}"; do - if FOUND=$(echo "$SYMBOLS" | grep "$BADSYMBOL"); then + if FOUND=$(grep "${BADSYMBOL}" < "${OUTPUT}/hyperkube-symbols"); then echo "Found bad symbol '${BADSYMBOL}':" echo "$FOUND" RESULT=1 diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index 3bffd1bf292..e2a7cb566ea 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -368,6 +368,8 @@ func dropDisabledFields( dropDisabledRunAsGroupField(podSpec, oldPodSpec) + dropDisabledGMSAFields(podSpec, oldPodSpec) + if !utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && !runtimeClassInUse(oldPodSpec) { // Set RuntimeClassName to nil only if feature is disabled and it is not used podSpec.RuntimeClassName = nil @@ -399,6 +401,39 @@ func dropDisabledRunAsGroupField(podSpec, oldPodSpec *api.PodSpec) { } } +// dropDisabledGMSAFields removes disabled fields related to Windows GMSA +// from the given PodSpec. +func dropDisabledGMSAFields(podSpec, oldPodSpec *api.PodSpec) { + if utilfeature.DefaultFeatureGate.Enabled(features.WindowsGMSA) || + gMSAFieldsInUse(oldPodSpec) { + return + } + + if podSpec.SecurityContext != nil { + dropDisabledGMSAFieldsFromWindowsSecurityOptions(podSpec.SecurityContext.WindowsOptions) + } + dropDisabledGMSAFieldsFromContainers(podSpec.Containers) + dropDisabledGMSAFieldsFromContainers(podSpec.InitContainers) +} + +// dropDisabledGMSAFieldsFromWindowsSecurityOptions removes disabled fields +// related to Windows GMSA from the given WindowsSecurityContextOptions. +func dropDisabledGMSAFieldsFromWindowsSecurityOptions(windowsOptions *api.WindowsSecurityContextOptions) { + if windowsOptions != nil { + windowsOptions.GMSACredentialSpecName = nil + windowsOptions.GMSACredentialSpec = nil + } +} + +// dropDisabledGMSAFieldsFromContainers removes disabled fields +func dropDisabledGMSAFieldsFromContainers(containers []api.Container) { + for i := range containers { + if containers[i].SecurityContext != nil { + dropDisabledGMSAFieldsFromWindowsSecurityOptions(containers[i].SecurityContext.WindowsOptions) + } + } +} + // dropDisabledProcMountField removes disabled fields from PodSpec related // to ProcMount only if it is not already used by the old spec func dropDisabledProcMountField(podSpec, oldPodSpec *api.PodSpec) { @@ -612,6 +647,44 @@ func runAsGroupInUse(podSpec *api.PodSpec) bool { return false } +// gMSAFieldsInUse returns true if the pod spec is non-nil and has one of any +// SecurityContext's GMSACredentialSpecName or GMSACredentialSpec fields set. +func gMSAFieldsInUse(podSpec *api.PodSpec) bool { + if podSpec == nil { + return false + } + + if podSpec.SecurityContext != nil && gMSAFieldsInUseInWindowsSecurityOptions(podSpec.SecurityContext.WindowsOptions) { + return true + } + + return gMSAFieldsInUseInAnyContainer(podSpec.Containers) || + gMSAFieldsInUseInAnyContainer(podSpec.InitContainers) +} + +// gMSAFieldsInUseInWindowsSecurityOptions returns true if the given WindowsSecurityContextOptions is +// non-nil and one of its GMSACredentialSpecName or GMSACredentialSpec fields is set. +func gMSAFieldsInUseInWindowsSecurityOptions(windowsOptions *api.WindowsSecurityContextOptions) bool { + if windowsOptions == nil { + return false + } + + return windowsOptions.GMSACredentialSpecName != nil || + windowsOptions.GMSACredentialSpec != nil +} + +// gMSAFieldsInUseInAnyContainer returns true if any of the given Containers has its +// SecurityContext's GMSACredentialSpecName or GMSACredentialSpec fields set. +func gMSAFieldsInUseInAnyContainer(containers []api.Container) bool { + for _, container := range containers { + if container.SecurityContext != nil && gMSAFieldsInUseInWindowsSecurityOptions(container.SecurityContext.WindowsOptions) { + return true + } + } + + return false +} + // subpathExprInUse returns true if the pod spec is non-nil and has a volume mount that makes use of the subPathExpr feature func subpathExprInUse(podSpec *api.PodSpec) bool { if podSpec == nil { diff --git a/pkg/api/pod/util_test.go b/pkg/api/pod/util_test.go index 0533a8861c7..f3523c5ada9 100644 --- a/pkg/api/pod/util_test.go +++ b/pkg/api/pod/util_test.go @@ -1359,6 +1359,208 @@ func TestDropRunAsGroup(t *testing.T) { } } +func TestDropGMSAFields(t *testing.T) { + defaultContainerSecurityContextFactory := func() *api.SecurityContext { + defaultProcMount := api.DefaultProcMount + return &api.SecurityContext{ProcMount: &defaultProcMount} + } + podWithoutWindowsOptionsFactory := func() *api.Pod { + return &api.Pod{ + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyNever, + SecurityContext: &api.PodSecurityContext{}, + Containers: []api.Container{{Name: "container1", Image: "testimage", SecurityContext: defaultContainerSecurityContextFactory()}}, + InitContainers: []api.Container{{Name: "initContainer1", Image: "testimage", SecurityContext: defaultContainerSecurityContextFactory()}}, + }, + } + } + + type podFactoryInfo struct { + description string + hasGMSAField bool + // this factory should generate the input pod whose spec will be fed to dropDisabledFields + podFactory func() *api.Pod + // this factory should generate the expected pod after the GMSA fields have been dropped + // we can't just use podWithoutWindowsOptionsFactory as is for this, since in some cases + // we'll be left with a WindowsSecurityContextOptions struct with no GMSA field set, as opposed + // to a nil pointer in the pod generated by podWithoutWindowsOptionsFactory + // if this field is not set, it will default to the podFactory + strippedPodFactory func() *api.Pod + } + podFactoryInfos := []podFactoryInfo{ + { + description: "does not have any GMSA field set", + hasGMSAField: false, + podFactory: podWithoutWindowsOptionsFactory, + }, + { + description: "has a pod-level WindowsSecurityContextOptions struct with no GMSA field set", + hasGMSAField: false, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }, + { + description: "has a WindowsSecurityContextOptions struct with no GMSA field set on a container", + hasGMSAField: false, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.Containers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }, + { + description: "has a WindowsSecurityContextOptions struct with no GMSA field set on an init container", + hasGMSAField: false, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }, + { + description: "is nil", + hasGMSAField: false, + podFactory: func() *api.Pod { return nil }, + }, + } + + toPtr := func(s string) *string { + return &s + } + addGMSACredentialSpecName := func(windowsOptions *api.WindowsSecurityContextOptions) { + windowsOptions.GMSACredentialSpecName = toPtr("dummy-gmsa-cred-spec-name") + } + addGMSACredentialSpec := func(windowsOptions *api.WindowsSecurityContextOptions) { + windowsOptions.GMSACredentialSpec = toPtr("dummy-gmsa-cred-spec-contents") + } + addBothGMSAFields := func(windowsOptions *api.WindowsSecurityContextOptions) { + addGMSACredentialSpecName(windowsOptions) + addGMSACredentialSpec(windowsOptions) + } + + for fieldName, windowsOptionsTransformingFunc := range map[string]func(*api.WindowsSecurityContextOptions){ + "GMSACredentialSpecName field": addGMSACredentialSpecName, + "GMSACredentialSpec field": addGMSACredentialSpec, + "both GMSA fields": addBothGMSAFields, + } { + // yes, these variables are indeed needed for the closure to work + // properly, please do NOT remove them + name := fieldName + transformingFunc := windowsOptionsTransformingFunc + + windowsOptionsWithGMSAFieldFactory := func() *api.WindowsSecurityContextOptions { + windowsOptions := &api.WindowsSecurityContextOptions{} + transformingFunc(windowsOptions) + return windowsOptions + } + + podFactoryInfos = append(podFactoryInfos, + podFactoryInfo{ + description: fmt.Sprintf("has %s in Pod", name), + hasGMSAField: true, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory() + return pod + }, + strippedPodFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }, + podFactoryInfo{ + description: fmt.Sprintf("has %s in Container", name), + hasGMSAField: true, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.Containers[0].SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory() + return pod + }, + strippedPodFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.Containers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }, + podFactoryInfo{ + description: fmt.Sprintf("has %s in InitContainer", name), + hasGMSAField: true, + podFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = windowsOptionsWithGMSAFieldFactory() + return pod + }, + strippedPodFactory: func() *api.Pod { + pod := podWithoutWindowsOptionsFactory() + pod.Spec.InitContainers[0].SecurityContext.WindowsOptions = &api.WindowsSecurityContextOptions{} + return pod + }, + }) + } + + for _, enabled := range []bool{true, false} { + for _, oldPodFactoryInfo := range podFactoryInfos { + for _, newPodFactoryInfo := range podFactoryInfos { + newPodHasGMSAField, newPod := newPodFactoryInfo.hasGMSAField, newPodFactoryInfo.podFactory() + if newPod == nil { + continue + } + oldPodHasGMSAField, oldPod := oldPodFactoryInfo.hasGMSAField, oldPodFactoryInfo.podFactory() + + t.Run(fmt.Sprintf("feature enabled=%v, old pod %s, new pod %s", enabled, oldPodFactoryInfo.description, newPodFactoryInfo.description), func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.WindowsGMSA, enabled)() + + var oldPodSpec *api.PodSpec + if oldPod != nil { + oldPodSpec = &oldPod.Spec + } + dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil) + + // old pod should never be changed + if !reflect.DeepEqual(oldPod, oldPodFactoryInfo.podFactory()) { + t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodFactoryInfo.podFactory())) + } + + switch { + case enabled || oldPodHasGMSAField: + // new pod should not be changed if the feature is enabled, or if the old pod had any GMSA field set + if !reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) { + t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodFactoryInfo.podFactory())) + } + case newPodHasGMSAField: + // new pod should be changed + if reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) { + t.Errorf("%v", oldPod) + t.Errorf("%v", newPod) + t.Errorf("new pod was not changed") + } + // new pod should not have any GMSA field set + var expectedStrippedPod *api.Pod + if newPodFactoryInfo.strippedPodFactory == nil { + expectedStrippedPod = newPodFactoryInfo.podFactory() + } else { + expectedStrippedPod = newPodFactoryInfo.strippedPodFactory() + } + + if !reflect.DeepEqual(newPod, expectedStrippedPod) { + t.Errorf("new pod had some GMSA field set: %v", diff.ObjectReflectDiff(newPod, expectedStrippedPod)) + } + default: + // new pod should not need to be changed + if !reflect.DeepEqual(newPod, newPodFactoryInfo.podFactory()) { + t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodFactoryInfo.podFactory())) + } + } + }) + } + } + } +} + func TestDropPodSysctls(t *testing.T) { podWithSysctls := func() *api.Pod { return &api.Pod{ diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 5b88d570706..fb0755e019c 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -4739,7 +4739,17 @@ type SELinuxOptions struct { // WindowsSecurityContextOptions contain Windows-specific options and credentials. type WindowsSecurityContextOptions struct { - // intentionally left empty for now + // GMSACredentialSpecName is the name of the GMSA credential spec to use. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + GMSACredentialSpecName *string + + // GMSACredentialSpec is where the GMSA admission webhook + // (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + // GMSA credential spec named by the GMSACredentialSpecName field. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + GMSACredentialSpec *string } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/core/v1/helper/helpers_test.go b/pkg/apis/core/v1/helper/helpers_test.go index f7b459245bd..9345be35af1 100644 --- a/pkg/apis/core/v1/helper/helpers_test.go +++ b/pkg/apis/core/v1/helper/helpers_test.go @@ -756,7 +756,7 @@ func TestMatchNodeSelectorTerms(t *testing.T) { // TestMatchNodeSelectorTermsStateless ensures MatchNodeSelectorTerms() // is invoked in a "stateless" manner, i.e. nodeSelectorTerms should NOT -// be deeply modifed after invoking +// be deeply modified after invoking func TestMatchNodeSelectorTermsStateless(t *testing.T) { type args struct { nodeSelectorTerms []v1.NodeSelectorTerm diff --git a/pkg/apis/core/v1/zz_generated.conversion.go b/pkg/apis/core/v1/zz_generated.conversion.go index 70952edd841..303da0cbd73 100644 --- a/pkg/apis/core/v1/zz_generated.conversion.go +++ b/pkg/apis/core/v1/zz_generated.conversion.go @@ -7599,6 +7599,8 @@ func Convert_core_WeightedPodAffinityTerm_To_v1_WeightedPodAffinityTerm(in *core } func autoConvert_v1_WindowsSecurityContextOptions_To_core_WindowsSecurityContextOptions(in *v1.WindowsSecurityContextOptions, out *core.WindowsSecurityContextOptions, s conversion.Scope) error { + out.GMSACredentialSpecName = (*string)(unsafe.Pointer(in.GMSACredentialSpecName)) + out.GMSACredentialSpec = (*string)(unsafe.Pointer(in.GMSACredentialSpec)) return nil } @@ -7608,6 +7610,8 @@ func Convert_v1_WindowsSecurityContextOptions_To_core_WindowsSecurityContextOpti } func autoConvert_core_WindowsSecurityContextOptions_To_v1_WindowsSecurityContextOptions(in *core.WindowsSecurityContextOptions, out *v1.WindowsSecurityContextOptions, s conversion.Scope) error { + out.GMSACredentialSpecName = (*string)(unsafe.Pointer(in.GMSACredentialSpecName)) + out.GMSACredentialSpec = (*string)(unsafe.Pointer(in.GMSACredentialSpec)) return nil } diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index bd54e385ac2..1f92208cd5a 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -3446,6 +3446,8 @@ func ValidatePodSecurityContext(securityContext *core.PodSecurityContext, spec * if len(securityContext.Sysctls) != 0 { allErrs = append(allErrs, validateSysctls(securityContext.Sysctls, fldPath.Child("sysctls"))...) } + + allErrs = append(allErrs, validateWindowsSecurityContextOptions(securityContext.WindowsOptions, fldPath.Child("windowsOptions"))...) } return allErrs @@ -5156,7 +5158,7 @@ func ValidateEndpointsUpdate(newEndpoints, oldEndpoints *core.Endpoints) field.E return allErrs } -// ValidateSecurityContext ensure the security context contains valid settings +// ValidateSecurityContext ensures the security context contains valid settings func ValidateSecurityContext(sc *core.SecurityContext, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} //this should only be true for testing since SecurityContext is defaulted by the core @@ -5202,6 +5204,42 @@ func ValidateSecurityContext(sc *core.SecurityContext, fldPath *field.Path) fiel } } + allErrs = append(allErrs, validateWindowsSecurityContextOptions(sc.WindowsOptions, fldPath.Child("windowsOptions"))...) + + return allErrs +} + +// maxGMSACredentialSpecLength is the max length, in bytes, for the actual contents +// of a GMSA cred spec. In general, those shouldn't be more than a few hundred bytes, +// so we want to give plenty of room here while still providing an upper bound. +const ( + maxGMSACredentialSpecLengthInKiB = 64 + maxGMSACredentialSpecLength = maxGMSACredentialSpecLengthInKiB * 1024 +) + +func validateWindowsSecurityContextOptions(windowsOptions *core.WindowsSecurityContextOptions, fieldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if windowsOptions == nil { + return allErrs + } + + if windowsOptions.GMSACredentialSpecName != nil { + // gmsaCredentialSpecName must be the name of a custom resource + for _, msg := range validation.IsDNS1123Subdomain(*windowsOptions.GMSACredentialSpecName) { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpecName"), windowsOptions.GMSACredentialSpecName, msg)) + } + } + + if windowsOptions.GMSACredentialSpec != nil { + if l := len(*windowsOptions.GMSACredentialSpec); l == 0 { + allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpec"), windowsOptions.GMSACredentialSpec, "gmsaCredentialSpec cannot be an empty string")) + } else if l > maxGMSACredentialSpecLength { + errMsg := fmt.Sprintf("gmsaCredentialSpec size must be under %d KiB", maxGMSACredentialSpecLengthInKiB) + allErrs = append(allErrs, field.Invalid(fieldPath.Child("gmsaCredentialSpec"), windowsOptions.GMSACredentialSpec, errMsg)) + } + } + return allErrs } diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 57b5a8866a7..2feccddb9b2 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -13205,3 +13205,77 @@ func TestValidateOrSetClientIPAffinityConfig(t *testing.T) { } } } + +func TestValidateWindowsSecurityContextOptions(t *testing.T) { + toPtr := func(s string) *string { + return &s + } + + testCases := []struct { + testName string + + windowsOptions *core.WindowsSecurityContextOptions + expectedErrorSubstring string + }{ + { + testName: "a nil pointer", + }, + { + testName: "an empty struct", + windowsOptions: &core.WindowsSecurityContextOptions{}, + }, + { + testName: "a valid input", + windowsOptions: &core.WindowsSecurityContextOptions{ + GMSACredentialSpecName: toPtr("dummy-gmsa-crep-spec-name"), + GMSACredentialSpec: toPtr("dummy-gmsa-crep-spec-contents"), + }, + }, + { + testName: "a GMSA cred spec name that is not a valid resource name", + windowsOptions: &core.WindowsSecurityContextOptions{ + // invalid because of the underscore + GMSACredentialSpecName: toPtr("not_a-valid-gmsa-crep-spec-name"), + }, + expectedErrorSubstring: dnsSubdomainLabelErrMsg, + }, + { + testName: "empty GMSA cred spec contents", + windowsOptions: &core.WindowsSecurityContextOptions{ + GMSACredentialSpec: toPtr(""), + }, + expectedErrorSubstring: "gmsaCredentialSpec cannot be an empty string", + }, + { + testName: "GMSA cred spec contents that are too long", + windowsOptions: &core.WindowsSecurityContextOptions{ + GMSACredentialSpec: toPtr(strings.Repeat("a", maxGMSACredentialSpecLength+1)), + }, + expectedErrorSubstring: "gmsaCredentialSpec size must be under", + }, + } + + for _, testCase := range testCases { + t.Run("validateWindowsSecurityContextOptions with"+testCase.testName, func(t *testing.T) { + errs := validateWindowsSecurityContextOptions(testCase.windowsOptions, field.NewPath("field")) + + switch len(errs) { + case 0: + if testCase.expectedErrorSubstring != "" { + t.Errorf("expected a failure containing the substring: %q", testCase.expectedErrorSubstring) + } + case 1: + if testCase.expectedErrorSubstring == "" { + t.Errorf("didn't expect a failure, got: %q", errs[0].Error()) + } else if !strings.Contains(errs[0].Error(), testCase.expectedErrorSubstring) { + t.Errorf("expected a failure with the substring %q, got %q instead", testCase.expectedErrorSubstring, errs[0].Error()) + } + default: + t.Errorf("got %d failures", len(errs)) + for i, err := range errs { + t.Errorf("error %d: %q", i, err.Error()) + } + } + }) + } +} diff --git a/pkg/apis/core/zz_generated.deepcopy.go b/pkg/apis/core/zz_generated.deepcopy.go index fd0aa11353a..b7b2282f62c 100644 --- a/pkg/apis/core/zz_generated.deepcopy.go +++ b/pkg/apis/core/zz_generated.deepcopy.go @@ -3464,7 +3464,7 @@ func (in *PodSecurityContext) DeepCopyInto(out *PodSecurityContext) { if in.WindowsOptions != nil { in, out := &in.WindowsOptions, &out.WindowsOptions *out = new(WindowsSecurityContextOptions) - **out = **in + (*in).DeepCopyInto(*out) } if in.RunAsUser != nil { in, out := &in.RunAsUser, &out.RunAsUser @@ -4646,7 +4646,7 @@ func (in *SecurityContext) DeepCopyInto(out *SecurityContext) { if in.WindowsOptions != nil { in, out := &in.WindowsOptions, &out.WindowsOptions *out = new(WindowsSecurityContextOptions) - **out = **in + (*in).DeepCopyInto(*out) } if in.RunAsUser != nil { in, out := &in.RunAsUser, &out.RunAsUser @@ -5475,6 +5475,16 @@ func (in *WeightedPodAffinityTerm) DeepCopy() *WeightedPodAffinityTerm { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WindowsSecurityContextOptions) DeepCopyInto(out *WindowsSecurityContextOptions) { *out = *in + if in.GMSACredentialSpecName != nil { + in, out := &in.GMSACredentialSpecName, &out.GMSACredentialSpecName + *out = new(string) + **out = **in + } + if in.GMSACredentialSpec != nil { + in, out := &in.GMSACredentialSpec, &out.GMSACredentialSpec + *out = new(string) + **out = **in + } return } diff --git a/pkg/capabilities/BUILD b/pkg/capabilities/BUILD index 1e00cb3c4c4..231f30ba4f7 100644 --- a/pkg/capabilities/BUILD +++ b/pkg/capabilities/BUILD @@ -1,10 +1,6 @@ package(default_visibility = ["//visibility:public"]) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -15,12 +11,6 @@ go_library( importpath = "k8s.io/kubernetes/pkg/capabilities", ) -go_test( - name = "go_default_test", - srcs = ["capabilities_test.go"], - embed = [":go_default_library"], -) - filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -33,3 +23,9 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["capabilities_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/capabilities/capabilities.go b/pkg/capabilities/capabilities.go index dc7f1e0e85c..eac7560ad35 100644 --- a/pkg/capabilities/capabilities.go +++ b/pkg/capabilities/capabilities.go @@ -61,10 +61,9 @@ func Initialize(c Capabilities) { } // Setup the capability set. It wraps Initialize for improving usability. -func Setup(allowPrivileged bool, privilegedSources PrivilegedSources, perConnectionBytesPerSec int64) { +func Setup(allowPrivileged bool, perConnectionBytesPerSec int64) { Initialize(Capabilities{ AllowPrivileged: allowPrivileged, - PrivilegedSources: privilegedSources, PerConnectionBandwidthLimitBytesPerSec: perConnectionBytesPerSec, }) } diff --git a/pkg/client/leaderelectionconfig/config.go b/pkg/client/leaderelectionconfig/config.go index 7be3d5aa598..223e24fa51b 100644 --- a/pkg/client/leaderelectionconfig/config.go +++ b/pkg/client/leaderelectionconfig/config.go @@ -17,18 +17,10 @@ limitations under the License. package leaderelectionconfig import ( - "time" - "github.com/spf13/pflag" componentbaseconfig "k8s.io/component-base/config" ) -const ( - // DefaultLeaseDuration defines a default duration of lease. - // TODO: This constant should move to the k8s.io/component-base/config package - DefaultLeaseDuration = 15 * time.Second -) - // BindFlags binds the LeaderElectionConfiguration struct fields to a flagset func BindFlags(l *componentbaseconfig.LeaderElectionConfiguration, fs *pflag.FlagSet) { fs.BoolVar(&l.LeaderElect, "leader-elect", l.LeaderElect, ""+ diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/pkg/cloudprovider/providers/openstack/openstack_routes.go index a1b447e37cf..3aa82dd70af 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_routes.go +++ b/pkg/cloudprovider/providers/openstack/openstack_routes.go @@ -111,12 +111,12 @@ func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, ne } unwinder := func() { - klog.V(4).Info("Reverting routes change to router ", router.ID) + klog.V(4).Infof("Reverting routes change to router %v", router.ID) _, err := routers.Update(network, router.ID, routers.UpdateOpts{ Routes: origRoutes, }).Extract() if err != nil { - klog.Warning("Unable to reset routes during error unwind: ", err) + klog.Warningf("Unable to reset routes during error unwind: %v", err) } } @@ -134,12 +134,12 @@ func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutron } unwinder := func() { - klog.V(4).Info("Reverting allowed-address-pairs change to port ", port.ID) + klog.V(4).Infof("Reverting allowed-address-pairs change to port %v", port.ID) _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ AllowedAddressPairs: &origPairs, }).Extract() if err != nil { - klog.Warning("Unable to reset allowed-address-pairs during error unwind: ", err) + klog.Warningf("Unable to reset allowed-address-pairs during error unwind: %v", err) } } @@ -200,7 +200,7 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s found := false for _, item := range port.AllowedAddressPairs { if item.IPAddress == route.DestinationCIDR { - klog.V(4).Info("Found existing allowed-address-pair: ", item) + klog.V(4).Infof("Found existing allowed-address-pair: %v", item) found = true break } diff --git a/pkg/controller/disruption/BUILD b/pkg/controller/disruption/BUILD index cca52c58c62..2bede6377a6 100644 --- a/pkg/controller/disruption/BUILD +++ b/pkg/controller/disruption/BUILD @@ -19,7 +19,9 @@ go_library( "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", @@ -34,6 +36,7 @@ go_library( "//staging/src/k8s.io/client-go/listers/apps/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/policy/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", @@ -49,13 +52,20 @@ go_test( "//pkg/apis/core/install:go_default_library", "//pkg/controller:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/api/autoscaling/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/meta/testrestmapper:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/scale/fake:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", diff --git a/pkg/controller/disruption/disruption.go b/pkg/controller/disruption/disruption.go index 9f34d927fa6..c2468e913fd 100644 --- a/pkg/controller/disruption/disruption.go +++ b/pkg/controller/disruption/disruption.go @@ -26,7 +26,9 @@ import ( policy "k8s.io/api/policy/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -41,6 +43,7 @@ import ( appsv1listers "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" policylisters "k8s.io/client-go/listers/policy/v1beta1" + scaleclient "k8s.io/client-go/scale" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -67,6 +70,9 @@ type updater func(*policy.PodDisruptionBudget) error type DisruptionController struct { kubeClient clientset.Interface + mapper apimeta.RESTMapper + + scaleNamespacer scaleclient.ScalesGetter pdbLister policylisters.PodDisruptionBudgetLister pdbListerSynced cache.InformerSynced @@ -105,7 +111,7 @@ type controllerAndScale struct { // podControllerFinder is a function type that maps a pod to a list of // controllers and their scale. -type podControllerFinder func(*v1.Pod) (*controllerAndScale, error) +type podControllerFinder func(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) func NewDisruptionController( podInformer coreinformers.PodInformer, @@ -115,6 +121,8 @@ func NewDisruptionController( dInformer appsv1informers.DeploymentInformer, ssInformer appsv1informers.StatefulSetInformer, kubeClient clientset.Interface, + restMapper apimeta.RESTMapper, + scaleNamespacer scaleclient.ScalesGetter, ) *DisruptionController { dc := &DisruptionController{ kubeClient: kubeClient, @@ -157,19 +165,19 @@ func NewDisruptionController( dc.ssLister = ssInformer.Lister() dc.ssListerSynced = ssInformer.Informer().HasSynced + dc.mapper = restMapper + dc.scaleNamespacer = scaleNamespacer + return dc } -// TODO(mml): When controllerRef is implemented (#2210), we *could* simply -// return controllers without their scales, and access scale type-generically -// via the scale subresource. That may not be as much of a win as it sounds, -// however. We are accessing everything through the pkg/client/cache API that -// we have to set up and tune to the types we know we'll be accessing anyway, -// and we may well need further tweaks just to be able to access scale -// subresources. +// The workload resources do implement the scale subresource, so it would +// be possible to only check the scale subresource here. But since there is no +// way to take advantage of listers with scale subresources, we use the workload +// resources directly and only fall back to the scale subresource when needed. func (dc *DisruptionController) finders() []podControllerFinder { return []podControllerFinder{dc.getPodReplicationController, dc.getPodDeployment, dc.getPodReplicaSet, - dc.getPodStatefulSet} + dc.getPodStatefulSet, dc.getScaleController} } var ( @@ -180,15 +188,12 @@ var ( ) // getPodReplicaSet finds a replicaset which has no matching deployments. -func (dc *DisruptionController) getPodReplicaSet(pod *v1.Pod) (*controllerAndScale, error) { - controllerRef := metav1.GetControllerOf(pod) - if controllerRef == nil { - return nil, nil +func (dc *DisruptionController) getPodReplicaSet(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) { + ok, err := verifyGroupKind(controllerRef, controllerKindRS.Kind, []string{"apps", "extensions"}) + if !ok || err != nil { + return nil, err } - if controllerRef.Kind != controllerKindRS.Kind { - return nil, nil - } - rs, err := dc.rsLister.ReplicaSets(pod.Namespace).Get(controllerRef.Name) + rs, err := dc.rsLister.ReplicaSets(namespace).Get(controllerRef.Name) if err != nil { // The only possible error is NotFound, which is ok here. return nil, nil @@ -204,16 +209,13 @@ func (dc *DisruptionController) getPodReplicaSet(pod *v1.Pod) (*controllerAndSca return &controllerAndScale{rs.UID, *(rs.Spec.Replicas)}, nil } -// getPodStatefulSet returns the statefulset managing the given pod. -func (dc *DisruptionController) getPodStatefulSet(pod *v1.Pod) (*controllerAndScale, error) { - controllerRef := metav1.GetControllerOf(pod) - if controllerRef == nil { - return nil, nil +// getPodStatefulSet returns the statefulset referenced by the provided controllerRef. +func (dc *DisruptionController) getPodStatefulSet(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) { + ok, err := verifyGroupKind(controllerRef, controllerKindSS.Kind, []string{"apps"}) + if !ok || err != nil { + return nil, err } - if controllerRef.Kind != controllerKindSS.Kind { - return nil, nil - } - ss, err := dc.ssLister.StatefulSets(pod.Namespace).Get(controllerRef.Name) + ss, err := dc.ssLister.StatefulSets(namespace).Get(controllerRef.Name) if err != nil { // The only possible error is NotFound, which is ok here. return nil, nil @@ -226,15 +228,12 @@ func (dc *DisruptionController) getPodStatefulSet(pod *v1.Pod) (*controllerAndSc } // getPodDeployments finds deployments for any replicasets which are being managed by deployments. -func (dc *DisruptionController) getPodDeployment(pod *v1.Pod) (*controllerAndScale, error) { - controllerRef := metav1.GetControllerOf(pod) - if controllerRef == nil { - return nil, nil +func (dc *DisruptionController) getPodDeployment(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) { + ok, err := verifyGroupKind(controllerRef, controllerKindRS.Kind, []string{"apps", "extensions"}) + if !ok || err != nil { + return nil, err } - if controllerRef.Kind != controllerKindRS.Kind { - return nil, nil - } - rs, err := dc.rsLister.ReplicaSets(pod.Namespace).Get(controllerRef.Name) + rs, err := dc.rsLister.ReplicaSets(namespace).Get(controllerRef.Name) if err != nil { // The only possible error is NotFound, which is ok here. return nil, nil @@ -246,8 +245,10 @@ func (dc *DisruptionController) getPodDeployment(pod *v1.Pod) (*controllerAndSca if controllerRef == nil { return nil, nil } - if controllerRef.Kind != controllerKindDep.Kind { - return nil, nil + + ok, err = verifyGroupKind(controllerRef, controllerKindDep.Kind, []string{"apps", "extensions"}) + if !ok || err != nil { + return nil, err } deployment, err := dc.dLister.Deployments(rs.Namespace).Get(controllerRef.Name) if err != nil { @@ -260,15 +261,12 @@ func (dc *DisruptionController) getPodDeployment(pod *v1.Pod) (*controllerAndSca return &controllerAndScale{deployment.UID, *(deployment.Spec.Replicas)}, nil } -func (dc *DisruptionController) getPodReplicationController(pod *v1.Pod) (*controllerAndScale, error) { - controllerRef := metav1.GetControllerOf(pod) - if controllerRef == nil { - return nil, nil +func (dc *DisruptionController) getPodReplicationController(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) { + ok, err := verifyGroupKind(controllerRef, controllerKindRC.Kind, []string{""}) + if !ok || err != nil { + return nil, err } - if controllerRef.Kind != controllerKindRC.Kind { - return nil, nil - } - rc, err := dc.rcLister.ReplicationControllers(pod.Namespace).Get(controllerRef.Name) + rc, err := dc.rcLister.ReplicationControllers(namespace).Get(controllerRef.Name) if err != nil { // The only possible error is NotFound, which is ok here. return nil, nil @@ -279,6 +277,55 @@ func (dc *DisruptionController) getPodReplicationController(pod *v1.Pod) (*contr return &controllerAndScale{rc.UID, *(rc.Spec.Replicas)}, nil } +func (dc *DisruptionController) getScaleController(controllerRef *metav1.OwnerReference, namespace string) (*controllerAndScale, error) { + gv, err := schema.ParseGroupVersion(controllerRef.APIVersion) + if err != nil { + return nil, err + } + + gk := schema.GroupKind{ + Group: gv.Group, + Kind: controllerRef.Kind, + } + + mapping, err := dc.mapper.RESTMapping(gk, gv.Version) + if err != nil { + return nil, err + } + gr := mapping.Resource.GroupResource() + + scale, err := dc.scaleNamespacer.Scales(namespace).Get(gr, controllerRef.Name) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + if scale.UID != controllerRef.UID { + return nil, nil + } + return &controllerAndScale{scale.UID, scale.Spec.Replicas}, nil +} + +func verifyGroupKind(controllerRef *metav1.OwnerReference, expectedKind string, expectedGroups []string) (bool, error) { + gv, err := schema.ParseGroupVersion(controllerRef.APIVersion) + if err != nil { + return false, err + } + + if controllerRef.Kind != expectedKind { + return false, nil + } + + for _, group := range expectedGroups { + if group == gv.Group { + return true, nil + } + } + + return false, nil +} + func (dc *DisruptionController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer dc.queue.ShutDown() @@ -583,10 +630,23 @@ func (dc *DisruptionController) getExpectedScale(pdb *policy.PodDisruptionBudget // 1. Find the controller for each pod. If any pod has 0 controllers, // that's an error. With ControllerRef, a pod can only have 1 controller. for _, pod := range pods { + controllerRef := metav1.GetControllerOf(pod) + if controllerRef == nil { + err = fmt.Errorf("found no controller ref for pod %q", pod.Name) + dc.recorder.Event(pdb, v1.EventTypeWarning, "NoControllerRef", err.Error()) + return + } + + // If we already know the scale of the controller there is no need to do anything. + if _, found := controllerScale[controllerRef.UID]; found { + continue + } + + // Check all the supported controllers to find the desired scale. foundController := false for _, finder := range dc.finders() { var controllerNScale *controllerAndScale - controllerNScale, err = finder(pod) + controllerNScale, err = finder(controllerRef, pod.Namespace) if err != nil { return } diff --git a/pkg/controller/disruption/disruption_test.go b/pkg/controller/disruption/disruption_test.go index 1f4e6c74efe..26341083317 100644 --- a/pkg/controller/disruption/disruption_test.go +++ b/pkg/controller/disruption/disruption_test.go @@ -23,13 +23,20 @@ import ( "time" apps "k8s.io/api/apps/v1" + autoscalingapi "k8s.io/api/autoscaling/v1" "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/meta/testrestmapper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/informers" + scalefake "k8s.io/client-go/scale/fake" + core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" _ "k8s.io/kubernetes/pkg/apis/core/install" @@ -90,6 +97,14 @@ type disruptionController struct { rsStore cache.Store dStore cache.Store ssStore cache.Store + + scaleClient *scalefake.FakeScaleClient +} + +var customGVK = schema.GroupVersionKind{ + Group: "custom.k8s.io", + Version: "v1", + Kind: "customresource", } func newFakeDisruptionController() (*disruptionController, *pdbStates) { @@ -97,6 +112,10 @@ func newFakeDisruptionController() (*disruptionController, *pdbStates) { informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc()) + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(customGVK, &v1.Service{}) + fakeScaleClient := &scalefake.FakeScaleClient{} + dc := NewDisruptionController( informerFactory.Core().V1().Pods(), informerFactory.Policy().V1beta1().PodDisruptionBudgets(), @@ -105,6 +124,8 @@ func newFakeDisruptionController() (*disruptionController, *pdbStates) { informerFactory.Apps().V1().Deployments(), informerFactory.Apps().V1().StatefulSets(), nil, + testrestmapper.TestOnlyStaticRESTMapper(scheme), + fakeScaleClient, ) dc.getUpdater = func() updater { return ps.Set } dc.podListerSynced = alwaysReady @@ -122,6 +143,7 @@ func newFakeDisruptionController() (*disruptionController, *pdbStates) { informerFactory.Apps().V1().ReplicaSets().Informer().GetStore(), informerFactory.Apps().V1().Deployments().Informer().GetStore(), informerFactory.Apps().V1().StatefulSets().Informer().GetStore(), + fakeScaleClient, }, ps } @@ -490,6 +512,52 @@ func TestReplicaSet(t *testing.T) { ps.VerifyPdbStatus(t, pdbName, 0, 1, 2, 10, map[string]metav1.Time{}) } +func TestScaleResource(t *testing.T) { + customResourceUID := uuid.NewUUID() + replicas := int32(10) + pods := int32(4) + maxUnavailable := int32(5) + + dc, ps := newFakeDisruptionController() + + dc.scaleClient.AddReactor("get", "customresources", func(action core.Action) (handled bool, ret runtime.Object, err error) { + obj := &autoscalingapi.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + UID: customResourceUID, + }, + Spec: autoscalingapi.ScaleSpec{ + Replicas: replicas, + }, + } + return true, obj, nil + }) + + pdb, pdbName := newMaxUnavailablePodDisruptionBudget(t, intstr.FromInt(int(maxUnavailable))) + add(t, dc.pdbStore, pdb) + + trueVal := true + for i := 0; i < int(pods); i++ { + pod, _ := newPod(t, fmt.Sprintf("pod-%d", i)) + pod.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: customGVK.Kind, + APIVersion: customGVK.GroupVersion().String(), + Controller: &trueVal, + UID: customResourceUID, + }, + }) + add(t, dc.podStore, pod) + } + + dc.sync(pdbName) + disruptionsAllowed := int32(0) + if replicas-pods < maxUnavailable { + disruptionsAllowed = maxUnavailable - (replicas - pods) + } + ps.VerifyPdbStatus(t, pdbName, disruptionsAllowed, pods, replicas-maxUnavailable, replicas, map[string]metav1.Time{}) +} + // Verify that multiple controllers doesn't allow the PDB to be set true. func TestMultipleControllers(t *testing.T) { const podCount = 2 @@ -759,3 +827,202 @@ func TestUpdateDisruptedPods(t *testing.T) { ps.VerifyPdbStatus(t, pdbName, 0, 1, 1, 3, map[string]metav1.Time{"p3": {Time: currentTime}}) } + +func TestBasicFinderFunctions(t *testing.T) { + dc, _ := newFakeDisruptionController() + + rs, _ := newReplicaSet(t, 10) + add(t, dc.rsStore, rs) + rc, _ := newReplicationController(t, 12) + add(t, dc.rcStore, rc) + ss, _ := newStatefulSet(t, 14) + add(t, dc.ssStore, ss) + + testCases := map[string]struct { + finderFunc podControllerFinder + apiVersion string + kind string + name string + uid types.UID + findsScale bool + expectedScale int32 + }{ + "replicaset controller with apps group": { + finderFunc: dc.getPodReplicaSet, + apiVersion: "apps/v1", + kind: controllerKindRS.Kind, + name: rs.Name, + uid: rs.UID, + findsScale: true, + expectedScale: 10, + }, + "replicaset controller with invalid group": { + finderFunc: dc.getPodReplicaSet, + apiVersion: "invalid/v1", + kind: controllerKindRS.Kind, + name: rs.Name, + uid: rs.UID, + findsScale: false, + }, + "replicationcontroller with empty group": { + finderFunc: dc.getPodReplicationController, + apiVersion: "/v1", + kind: controllerKindRC.Kind, + name: rc.Name, + uid: rc.UID, + findsScale: true, + expectedScale: 12, + }, + "replicationcontroller with invalid group": { + finderFunc: dc.getPodReplicationController, + apiVersion: "apps/v1", + kind: controllerKindRC.Kind, + name: rc.Name, + uid: rc.UID, + findsScale: false, + }, + "statefulset controller with extensions group": { + finderFunc: dc.getPodStatefulSet, + apiVersion: "apps/v1", + kind: controllerKindSS.Kind, + name: ss.Name, + uid: ss.UID, + findsScale: true, + expectedScale: 14, + }, + "statefulset controller with invalid kind": { + finderFunc: dc.getPodStatefulSet, + apiVersion: "apps/v1", + kind: controllerKindRS.Kind, + name: ss.Name, + uid: ss.UID, + findsScale: false, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + controllerRef := &metav1.OwnerReference{ + APIVersion: tc.apiVersion, + Kind: tc.kind, + Name: tc.name, + UID: tc.uid, + } + + controllerAndScale, _ := tc.finderFunc(controllerRef, metav1.NamespaceDefault) + + if controllerAndScale == nil { + if tc.findsScale { + t.Error("Expected scale, but got nil") + } + return + } + + if got, want := controllerAndScale.scale, tc.expectedScale; got != want { + t.Errorf("Expected scale %d, but got %d", want, got) + } + + if got, want := controllerAndScale.UID, tc.uid; got != want { + t.Errorf("Expected uid %s, but got %s", want, got) + } + }) + } +} + +func TestDeploymentFinderFunction(t *testing.T) { + labels := map[string]string{ + "foo": "bar", + } + + testCases := map[string]struct { + rsApiVersion string + rsKind string + depApiVersion string + depKind string + findsScale bool + expectedScale int32 + }{ + "happy path": { + rsApiVersion: "apps/v1", + rsKind: controllerKindRS.Kind, + depApiVersion: "extensions/v1", + depKind: controllerKindDep.Kind, + findsScale: true, + expectedScale: 10, + }, + "invalid rs apiVersion": { + rsApiVersion: "invalid/v1", + rsKind: controllerKindRS.Kind, + depApiVersion: "apps/v1", + depKind: controllerKindDep.Kind, + findsScale: false, + }, + "invalid rs kind": { + rsApiVersion: "apps/v1", + rsKind: "InvalidKind", + depApiVersion: "apps/v1", + depKind: controllerKindDep.Kind, + findsScale: false, + }, + "invalid deployment apiVersion": { + rsApiVersion: "extensions/v1", + rsKind: controllerKindRS.Kind, + depApiVersion: "deployment/v1", + depKind: controllerKindDep.Kind, + findsScale: false, + }, + "invalid deployment kind": { + rsApiVersion: "apps/v1", + rsKind: controllerKindRS.Kind, + depApiVersion: "extensions/v1", + depKind: "InvalidKind", + findsScale: false, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + dc, _ := newFakeDisruptionController() + + dep, _ := newDeployment(t, 10) + dep.Spec.Selector = newSel(labels) + add(t, dc.dStore, dep) + + rs, _ := newReplicaSet(t, 5) + rs.Labels = labels + trueVal := true + rs.OwnerReferences = append(rs.OwnerReferences, metav1.OwnerReference{ + APIVersion: tc.depApiVersion, + Kind: tc.depKind, + Name: dep.Name, + UID: dep.UID, + Controller: &trueVal, + }) + add(t, dc.rsStore, rs) + + controllerRef := &metav1.OwnerReference{ + APIVersion: tc.rsApiVersion, + Kind: tc.rsKind, + Name: rs.Name, + UID: rs.UID, + } + + controllerAndScale, _ := dc.getPodDeployment(controllerRef, metav1.NamespaceDefault) + + if controllerAndScale == nil { + if tc.findsScale { + t.Error("Expected scale, but got nil") + } + return + } + + if got, want := controllerAndScale.scale, tc.expectedScale; got != want { + t.Errorf("Expected scale %d, but got %d", want, got) + } + + if got, want := controllerAndScale.UID, dep.UID; got != want { + t.Errorf("Expected uid %s, but got %s", want, got) + } + }) + } +} diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index e3cecd9fc2d..665c726f77b 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -631,16 +631,15 @@ func pastBackoffLimitOnFailure(job *batch.Job, pods []*v1.Pod) bool { result := int32(0) for i := range pods { po := pods[i] - if po.Status.Phase != v1.PodRunning { - continue - } - for j := range po.Status.InitContainerStatuses { - stat := po.Status.InitContainerStatuses[j] - result += stat.RestartCount - } - for j := range po.Status.ContainerStatuses { - stat := po.Status.ContainerStatuses[j] - result += stat.RestartCount + if po.Status.Phase == v1.PodRunning || po.Status.Phase == v1.PodPending { + for j := range po.Status.InitContainerStatuses { + stat := po.Status.InitContainerStatuses[j] + result += stat.RestartCount + } + for j := range po.Status.ContainerStatuses { + stat := po.Status.ContainerStatuses[j] + result += stat.RestartCount + } } } if *job.Spec.BackoffLimit == 0 { diff --git a/pkg/controller/job/job_controller_test.go b/pkg/controller/job/job_controller_test.go index 8be2deed3e9..bc5a493cefb 100644 --- a/pkg/controller/job/job_controller_test.go +++ b/pkg/controller/job/job_controller_test.go @@ -1414,6 +1414,7 @@ func TestJobBackoffForOnFailure(t *testing.T) { // pod setup jobKeyForget bool restartCounts []int32 + podPhase v1.PodPhase // expectations expectedActive int32 @@ -1424,32 +1425,47 @@ func TestJobBackoffForOnFailure(t *testing.T) { }{ "backoffLimit 0 should have 1 pod active": { 1, 1, 0, - true, []int32{0}, + true, []int32{0}, v1.PodRunning, 1, 0, 0, nil, "", }, "backoffLimit 1 with restartCount 0 should have 1 pod active": { 1, 1, 1, - true, []int32{0}, + true, []int32{0}, v1.PodRunning, 1, 0, 0, nil, "", }, - "backoffLimit 1 with restartCount 1 should have 0 pod active": { + "backoffLimit 1 with restartCount 1 and podRunning should have 0 pod active": { 1, 1, 1, - true, []int32{1}, + true, []int32{1}, v1.PodRunning, 0, 0, 1, &jobConditionFailed, "BackoffLimitExceeded", }, - "too many job failures - single pod": { + "backoffLimit 1 with restartCount 1 and podPending should have 0 pod active": { + 1, 1, 1, + true, []int32{1}, v1.PodPending, + 0, 0, 1, &jobConditionFailed, "BackoffLimitExceeded", + }, + "too many job failures with podRunning - single pod": { 1, 5, 2, - true, []int32{2}, + true, []int32{2}, v1.PodRunning, 0, 0, 1, &jobConditionFailed, "BackoffLimitExceeded", }, - "too many job failures - multiple pods": { + "too many job failures with podPending - single pod": { + 1, 5, 2, + true, []int32{2}, v1.PodPending, + 0, 0, 1, &jobConditionFailed, "BackoffLimitExceeded", + }, + "too many job failures with podRunning - multiple pods": { 2, 5, 2, - true, []int32{1, 1}, + true, []int32{1, 1}, v1.PodRunning, + 0, 0, 2, &jobConditionFailed, "BackoffLimitExceeded", + }, + "too many job failures with podPending - multiple pods": { + 2, 5, 2, + true, []int32{1, 1}, v1.PodPending, 0, 0, 2, &jobConditionFailed, "BackoffLimitExceeded", }, "not enough failures": { 2, 5, 3, - true, []int32{1, 1}, + true, []int32{1, 1}, v1.PodRunning, 2, 0, 0, nil, "", }, } @@ -1474,7 +1490,7 @@ func TestJobBackoffForOnFailure(t *testing.T) { job.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyOnFailure sharedInformerFactory.Batch().V1().Jobs().Informer().GetIndexer().Add(job) podIndexer := sharedInformerFactory.Core().V1().Pods().Informer().GetIndexer() - for i, pod := range newPodList(int32(len(tc.restartCounts)), v1.PodRunning, job) { + for i, pod := range newPodList(int32(len(tc.restartCounts)), tc.podPhase, job) { pod.Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: tc.restartCounts[i]}} podIndexer.Add(&pod) } diff --git a/pkg/controller/nodelifecycle/node_lifecycle_controller.go b/pkg/controller/nodelifecycle/node_lifecycle_controller.go index 98e1a63934b..68647db16f6 100644 --- a/pkg/controller/nodelifecycle/node_lifecycle_controller.go +++ b/pkg/controller/nodelifecycle/node_lifecycle_controller.go @@ -862,7 +862,7 @@ func (nc *Controller) tryUpdateNodeHealth(node *v1.Node) (time.Duration, v1.Node transitionTime = savedNodeHealth.readyTransitionTimestamp } if klog.V(5) { - klog.V(5).Infof("Node %s ReadyCondition updated. Updating timestamp: %+v vs %+v.", node.Name, savedNodeHealth.status, node.Status) + klog.Infof("Node %s ReadyCondition updated. Updating timestamp: %+v vs %+v.", node.Name, savedNodeHealth.status, node.Status) } else { klog.V(3).Infof("Node %s ReadyCondition updated. Updating timestamp.", node.Name) } diff --git a/pkg/controller/serviceaccount/serviceaccounts_controller.go b/pkg/controller/serviceaccount/serviceaccounts_controller.go index 5f10312ae17..52d50d7fb15 100644 --- a/pkg/controller/serviceaccount/serviceaccounts_controller.go +++ b/pkg/controller/serviceaccount/serviceaccounts_controller.go @@ -199,8 +199,7 @@ func (c *ServiceAccountsController) syncNamespace(key string) error { } createFailures := []error{} - for i := range c.serviceAccountsToEnsure { - sa := c.serviceAccountsToEnsure[i] + for _, sa := range c.serviceAccountsToEnsure { switch _, err := c.saLister.ServiceAccounts(ns.Name).Get(sa.Name); { case err == nil: continue diff --git a/pkg/controller/util/node/BUILD b/pkg/controller/util/node/BUILD index 4b860e68ffc..c875ccf674c 100644 --- a/pkg/controller/util/node/BUILD +++ b/pkg/controller/util/node/BUILD @@ -6,6 +6,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/controller/util/node", visibility = ["//visibility:public"], deps = [ + "//pkg/api/v1/pod:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/controller:go_default_library", "//pkg/kubelet/util/format:go_default_library", diff --git a/pkg/controller/util/node/controller_utils.go b/pkg/controller/util/node/controller_utils.go index d919e7177a2..da17b5696b5 100644 --- a/pkg/controller/util/node/controller_utils.go +++ b/pkg/controller/util/node/controller_utils.go @@ -32,6 +32,7 @@ import ( "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" appsv1listers "k8s.io/client-go/listers/apps/v1" + utilpod "k8s.io/kubernetes/pkg/api/v1/pod" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubelet/util/format" @@ -134,9 +135,12 @@ func MarkAllPodsNotReady(kubeClient clientset.Interface, node *v1.Node) error { continue } - for i, cond := range pod.Status.Conditions { + for _, cond := range pod.Status.Conditions { if cond.Type == v1.PodReady { - pod.Status.Conditions[i].Status = v1.ConditionFalse + cond.Status = v1.ConditionFalse + if !utilpod.UpdatePodCondition(&pod.Status, &cond) { + break + } klog.V(2).Infof("Updating ready status of pod %v to false", pod.Name) _, err := kubeClient.CoreV1().Pods(pod.Namespace).UpdateStatus(&pod) if err != nil { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 12e3fba4941..0bf79e5ddd7 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -532,7 +532,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS apiextensionsfeatures.CustomResourceValidation: {Default: true, PreRelease: featuregate.Beta}, apiextensionsfeatures.CustomResourceSubresources: {Default: true, PreRelease: featuregate.Beta}, apiextensionsfeatures.CustomResourceWebhookConversion: {Default: false, PreRelease: featuregate.Alpha}, - apiextensionsfeatures.CustomResourcePublishOpenAPI: {Default: false, PreRelease: featuregate.Alpha}, + apiextensionsfeatures.CustomResourcePublishOpenAPI: {Default: true, PreRelease: featuregate.Beta}, // features that enable backwards compatibility but are scheduled to be removed // ... diff --git a/pkg/kubeapiserver/server/insecure_handler.go b/pkg/kubeapiserver/server/insecure_handler.go index 05937e32d7e..850353538b1 100644 --- a/pkg/kubeapiserver/server/insecure_handler.go +++ b/pkg/kubeapiserver/server/insecure_handler.go @@ -24,10 +24,6 @@ import ( genericfilters "k8s.io/apiserver/pkg/server/filters" ) -// DeprecatedInsecureServingInfo is required to serve http. HTTP does NOT include authentication or authorization. -// You shouldn't be using this. It makes sig-auth sad. -// DeprecatedInsecureServingInfo *ServingInfo - // BuildInsecureHandlerChain sets up the server to listen to http. Should be removed. func BuildInsecureHandlerChain(apiHandler http.Handler, c *server.Config) http.Handler { handler := apiHandler diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index e3188430b95..2a1c8b05d8e 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -38,7 +38,6 @@ go_library( "//pkg/apis/core/v1:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", - "//pkg/capabilities:go_default_library", "//pkg/features:go_default_library", "//pkg/fieldpath:go_default_library", "//pkg/kubelet/apis:go_default_library", @@ -99,13 +98,13 @@ go_library( "//pkg/scheduler/api:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/security/podsecuritypolicy/sysctl:go_default_library", - "//pkg/securitycontext:go_default_library", "//pkg/util/dbus:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/util/node:go_default_library", "//pkg/util/oom:go_default_library", "//pkg/util/removeall:go_default_library", + "//pkg/util/selinux:go_default_library", "//pkg/util/taints:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/csi:go_default_library", @@ -175,7 +174,6 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core/install:go_default_library", - "//pkg/capabilities:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 079cb192534..cfbf6e67bfa 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -291,12 +291,12 @@ type KubeletConfiguration struct { /* the following fields are meant for Node Allocatable */ - // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pids=100) pairs + // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pid=100) pairs // that describe resources reserved for non-kubernetes components. // Currently only cpu and memory are supported. // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. SystemReserved map[string]string - // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pids=100) pairs + // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G,pid=100) pairs // that describe resources reserved for kubernetes system components. // Currently cpu, memory and local ephemeral storage for root file system are supported. // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 804c455b15a..c2e456aa108 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -166,6 +166,7 @@ filegroup( "//pkg/kubelet/cm/cpumanager:all-srcs", "//pkg/kubelet/cm/cpuset:all-srcs", "//pkg/kubelet/cm/devicemanager:all-srcs", + "//pkg/kubelet/cm/topologymanager/socketmask:all-srcs", "//pkg/kubelet/cm/util:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 5427183cb3c..315214bc634 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" // TODO: Migrate kubelet to either use its own internal objects or client library. - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" internalapi "k8s.io/cri-api/pkg/apis" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" @@ -104,6 +104,10 @@ type ContainerManager interface { // GetDevices returns information about the devices assigned to pods and containers GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices + + // ShouldResetExtendedResourceCapacity returns whether or not the extended resources should be zeroed, + // due to node recreation. + ShouldResetExtendedResourceCapacity() bool } type NodeConfig struct { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 79e647b8c97..1c4688ccf7f 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -34,7 +34,7 @@ import ( "github.com/opencontainers/runc/libcontainer/configs" "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -897,3 +897,7 @@ func (cm *containerManagerImpl) GetDevicePluginResourceCapacity() (v1.ResourceLi func (cm *containerManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { return cm.deviceManager.GetDevices(podUID, containerName) } + +func (cm *containerManagerImpl) ShouldResetExtendedResourceCapacity() bool { + return cm.deviceManager.ShouldResetExtendedResourceCapacity() +} diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index db5de157f98..4ea918511c3 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -17,7 +17,7 @@ limitations under the License. package cm import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/klog" "k8s.io/apimachinery/pkg/api/resource" @@ -32,7 +32,9 @@ import ( schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) -type containerManagerStub struct{} +type containerManagerStub struct { + shouldResetExtendedResourceCapacity bool +} var _ ContainerManager = &containerManagerStub{} @@ -110,6 +112,14 @@ func (cm *containerManagerStub) GetDevices(_, _ string) []*podresourcesapi.Conta return nil } -func NewStubContainerManager() ContainerManager { - return &containerManagerStub{} +func (cm *containerManagerStub) ShouldResetExtendedResourceCapacity() bool { + return cm.shouldResetExtendedResourceCapacity +} + +func NewStubContainerManager() ContainerManager { + return &containerManagerStub{shouldResetExtendedResourceCapacity: false} +} + +func NewStubContainerManagerWithExtendedResource(shouldResetExtendedResourceCapacity bool) ContainerManager { + return &containerManagerStub{shouldResetExtendedResourceCapacity: shouldResetExtendedResourceCapacity} } diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index 092249af9bb..d2e0574b7f2 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -24,7 +24,7 @@ package cm import ( "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" @@ -171,3 +171,7 @@ func (cm *containerManagerImpl) GetPodCgroupRoot() string { func (cm *containerManagerImpl) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { return nil } + +func (cm *containerManagerImpl) ShouldResetExtendedResourceCapacity() bool { + return false +} diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD index f45f3fc9897..796bfe6a059 100644 --- a/pkg/kubelet/cm/devicemanager/BUILD +++ b/pkg/kubelet/cm/devicemanager/BUILD @@ -14,6 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apis/core/v1/helper:go_default_library", + "//pkg/features:go_default_library", "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", @@ -26,9 +27,11 @@ go_library( "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/util/pluginwatcher:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", + "//pkg/util/selinux:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index 355a223a4c6..8fa58c4782d 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -28,10 +28,12 @@ import ( "google.golang.org/grpc" "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/features" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -42,6 +44,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" watcher "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" + "k8s.io/kubernetes/pkg/util/selinux" ) // ActivePodsFunc is a function that returns a list of pods to reconcile. @@ -206,6 +209,11 @@ func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.Sourc socketPath := filepath.Join(m.socketdir, m.socketname) os.MkdirAll(m.socketdir, 0755) + if selinux.SELinuxEnabled() { + if err := selinux.SetFileLabel(m.socketdir, config.KubeletPluginsDirSELinuxLabel); err != nil { + klog.Warningf("Unprivileged containerized plugins might not work. Could not set selinux context on %s: %v", m.socketdir, err) + } + } // Removes all stale sockets in m.socketdir. Device plugins can monitor // this and use it as a signal to re-register with the new Kubelet. @@ -832,3 +840,17 @@ func (m *ManagerImpl) GetDevices(podUID, containerName string) []*podresourcesap defer m.mutex.Unlock() return m.podDevices.getContainerDevices(podUID, containerName) } + +// ShouldResetExtendedResourceCapacity returns whether the extended resources should be zeroed or not, +// depending on whether the node has been recreated. Absence of the checkpoint file strongly indicates the node +// has been recreated. +func (m *ManagerImpl) ShouldResetExtendedResourceCapacity() bool { + if utilfeature.DefaultFeatureGate.Enabled(features.DevicePlugins) { + checkpoints, err := m.checkpointManager.ListCheckpoints() + if err != nil { + return false + } + return len(checkpoints) == 0 + } + return false +} diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go index a4309c78a40..b24c116c10f 100644 --- a/pkg/kubelet/cm/devicemanager/manager_stub.go +++ b/pkg/kubelet/cm/devicemanager/manager_stub.go @@ -17,7 +17,7 @@ limitations under the License. package devicemanager import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/lifecycle" @@ -67,3 +67,8 @@ func (h *ManagerStub) GetWatcherHandler() pluginwatcher.PluginHandler { func (h *ManagerStub) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { return nil } + +// ShouldResetExtendedResourceCapacity returns false +func (h *ManagerStub) ShouldResetExtendedResourceCapacity() bool { + return false +} diff --git a/pkg/kubelet/cm/devicemanager/manager_test.go b/pkg/kubelet/cm/devicemanager/manager_test.go index 6cd969412cf..a3c55dd15fb 100644 --- a/pkg/kubelet/cm/devicemanager/manager_test.go +++ b/pkg/kubelet/cm/devicemanager/manager_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -946,6 +946,45 @@ func TestDevicePreStartContainer(t *testing.T) { as.Equal(len(runContainerOpts.Envs), len(expectedResp.Envs)) } +func TestResetExtendedResource(t *testing.T) { + as := assert.New(t) + tmpDir, err := ioutil.TempDir("", "checkpoint") + as.Nil(err) + ckm, err := checkpointmanager.NewCheckpointManager(tmpDir) + as.Nil(err) + testManager := &ManagerImpl{ + endpoints: make(map[string]endpointInfo), + healthyDevices: make(map[string]sets.String), + unhealthyDevices: make(map[string]sets.String), + allocatedDevices: make(map[string]sets.String), + podDevices: make(podDevices), + checkpointManager: ckm, + } + + extendedResourceName := "domain.com/resource" + testManager.podDevices.insert("pod", "con", extendedResourceName, + constructDevices([]string{"dev1"}), + constructAllocResp(map[string]string{"/dev/dev1": "/dev/dev1"}, + map[string]string{"/home/lib1": "/usr/lib1"}, map[string]string{})) + + testManager.healthyDevices[extendedResourceName] = sets.NewString() + testManager.healthyDevices[extendedResourceName].Insert("dev1") + // checkpoint is present, indicating node hasn't been recreated + err = testManager.writeCheckpoint() + as.Nil(err) + + as.False(testManager.ShouldResetExtendedResourceCapacity()) + + // checkpoint is absent, representing node recreation + ckpts, err := ckm.ListCheckpoints() + as.Nil(err) + for _, ckpt := range ckpts { + err = ckm.RemoveCheckpoint(ckpt) + as.Nil(err) + } + as.True(testManager.ShouldResetExtendedResourceCapacity()) +} + func allocateStubFunc() func(devs []string) (*pluginapi.AllocateResponse, error) { return func(devs []string) (*pluginapi.AllocateResponse, error) { resp := new(pluginapi.ContainerAllocateResponse) diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index a420cf6541f..0904db5f1b3 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -19,7 +19,7 @@ package devicemanager import ( "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -58,6 +58,11 @@ type Manager interface { // GetDevices returns information about the devices assigned to pods and containers GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices + + // ShouldResetExtendedResourceCapacity returns whether the extended resources should be reset or not, + // depending on the checkpoint file availability. Absence of the checkpoint file strongly indicates + // the node has been recreated. + ShouldResetExtendedResourceCapacity() bool } // DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices. diff --git a/pkg/kubelet/cm/topologymanager/socketmask/BUILD b/pkg/kubelet/cm/topologymanager/socketmask/BUILD new file mode 100644 index 00000000000..25bd381e862 --- /dev/null +++ b/pkg/kubelet/cm/topologymanager/socketmask/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["socketmask.go"], + importpath = "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/socketmask", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["socketmask_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/kubelet/cm/topologymanager/socketmask/socketmask.go b/pkg/kubelet/cm/topologymanager/socketmask/socketmask.go new file mode 100644 index 00000000000..8e74ce0291a --- /dev/null +++ b/pkg/kubelet/cm/topologymanager/socketmask/socketmask.go @@ -0,0 +1,152 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package socketmask + +import ( + "fmt" +) + +//SocketMask interface allows hint providers to create SocketMasks for TopologyHints +type SocketMask interface { + Add(sockets ...int) error + Remove(sockets ...int) error + And(masks ...SocketMask) + Or(masks ...SocketMask) + Clear() + Fill() + IsEqual(mask SocketMask) bool + IsEmpty() bool + IsSet(socket int) bool + String() string + Count() int + GetSockets() []int +} + +type socketMask uint64 + +//NewSocketMask creates a new SocketMask +func NewSocketMask(sockets ...int) (SocketMask, error) { + s := socketMask(0) + err := (&s).Add(sockets...) + if err != nil { + return nil, err + } + return &s, nil +} + +//Add adds the sockets with topology affinity to the SocketMask +func (s *socketMask) Add(sockets ...int) error { + mask := *s + for _, i := range sockets { + if i < 0 || i >= 64 { + return fmt.Errorf("socket number must be in range 0-63") + } + mask |= 1 << uint64(i) + } + *s = mask + return nil +} + +//Remove removes specified sockets from SocketMask +func (s *socketMask) Remove(sockets ...int) error { + mask := *s + for _, i := range sockets { + if i < 0 || i >= 64 { + return fmt.Errorf("socket number must be in range 0-63") + } + mask &^= 1 << uint64(i) + } + *s = mask + return nil +} + +//And performs and operation on all bits in masks +func (s *socketMask) And(masks ...SocketMask) { + for _, m := range masks { + *s &= *m.(*socketMask) + } +} + +//Or performs or operation on all bits in masks +func (s *socketMask) Or(masks ...SocketMask) { + for _, m := range masks { + *s |= *m.(*socketMask) + } +} + +//Clear resets all bits in mask to zero +func (s *socketMask) Clear() { + *s = 0 +} + +//Fill sets all bits in mask to one +func (s *socketMask) Fill() { + *s = socketMask(^uint64(0)) +} + +//IsEmpty checks mask to see if all bits are zero +func (s *socketMask) IsEmpty() bool { + return *s == 0 +} + +//IsSet checks socket in mask to see if bit is set to one +func (s *socketMask) IsSet(socket int) bool { + if socket < 0 || socket >= 64 { + return false + } + return (*s & (1 << uint64(socket))) > 0 +} + +//IsEqual checks if masks are equal +func (s *socketMask) IsEqual(mask SocketMask) bool { + return *s == *mask.(*socketMask) +} + +//String converts mask to string +func (s *socketMask) String() string { + str := "" + for i := uint64(0); i < 64; i++ { + if (*s & (1 << i)) > 0 { + str += "1" + } else { + str += "0" + } + } + return str +} + +//Count counts number of bits in mask set to one +func (s *socketMask) Count() int { + count := 0 + for i := uint64(0); i < 64; i++ { + if (*s & (1 << i)) > 0 { + count++ + } + } + return count +} + +//GetSockets returns each socket number with bits set to one +func (s *socketMask) GetSockets() []int { + var sockets []int + for i := uint64(0); i < 64; i++ { + if (*s & (1 << i)) > 0 { + sockets = append(sockets, int(i)) + } + } + return sockets +} diff --git a/pkg/kubelet/cm/topologymanager/socketmask/socketmask_test.go b/pkg/kubelet/cm/topologymanager/socketmask/socketmask_test.go new file mode 100644 index 00000000000..01df68999c0 --- /dev/null +++ b/pkg/kubelet/cm/topologymanager/socketmask/socketmask_test.go @@ -0,0 +1,290 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package socketmask + +import ( + "reflect" + "testing" +) + +func TestNewSocketMask(t *testing.T) { + tcases := []struct { + name string + socket int + expectedMask string + }{ + { + name: "New SocketMask with socket 0 set", + socket: int(0), + expectedMask: "1000000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + sm, _ := NewSocketMask(0) + if sm.String() != tc.expectedMask { + t.Errorf("Expected mask to be %v, got %v", tc.expectedMask, sm) + } + } +} + +func TestAdd(t *testing.T) { + tcases := []struct { + name string + firstSocket int + secondSocket int + expectedMask string + }{ + { + name: "Reset bit 1 SocketMask to 0", + firstSocket: 0, + secondSocket: 1, + expectedMask: "1100000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask() + mask.Add(tc.firstSocket, tc.secondSocket) + if mask.String() != tc.expectedMask { + t.Errorf("Expected mask to be %v, got %v", tc.expectedMask, mask) + } + } +} + +func TestRemove(t *testing.T) { + tcases := []struct { + name string + firstSocketSet int + secondSocketSet int + firstSocketRemove int + expectedMask string + }{ + { + name: "Reset bit 1 SocketMask to 0", + firstSocketSet: 0, + secondSocketSet: 1, + firstSocketRemove: 0, + expectedMask: "0100000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.firstSocketSet, tc.secondSocketSet) + mask.Remove(tc.firstSocketRemove) + if mask.String() != tc.expectedMask { + t.Errorf("Expected mask to be %v, got %v", tc.expectedMask, mask) + } + } +} + +func TestAnd(t *testing.T) { + tcases := []struct { + name string + firstMaskBit int + secondMaskBit int + andMask string + }{ + { + name: "And socket masks", + firstMaskBit: 0, + secondMaskBit: 0, + andMask: "1000000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + firstMask, _ := NewSocketMask(tc.firstMaskBit) + secondMask, _ := NewSocketMask(tc.secondMaskBit) + firstMask.And(secondMask) + if firstMask.String() != string(tc.andMask) { + t.Errorf("Expected mask to be %v, got %v", tc.andMask, firstMask) + } + } +} + +func TestOr(t *testing.T) { + tcases := []struct { + name string + firstMaskBit int + secondMaskBit int + orMask string + }{ + { + name: "Or socket masks", + firstMaskBit: int(0), + secondMaskBit: int(1), + orMask: "1100000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + firstMask, _ := NewSocketMask(tc.firstMaskBit) + secondMask, _ := NewSocketMask(tc.secondMaskBit) + firstMask.Or(secondMask) + if firstMask.String() != string(tc.orMask) { + t.Errorf("Expected mask to be %v, got %v", tc.orMask, firstMask) + } + } +} + +func TestClear(t *testing.T) { + tcases := []struct { + name string + firstBit int + secondBit int + clearedMask string + }{ + { + name: "Clear socket masks", + firstBit: int(0), + secondBit: int(1), + clearedMask: "0000000000000000000000000000000000000000000000000000000000000000", + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.firstBit, tc.secondBit) + mask.Clear() + if mask.String() != string(tc.clearedMask) { + t.Errorf("Expected mask to be %v, got %v", tc.clearedMask, mask) + } + } +} + +func TestFill(t *testing.T) { + tcases := []struct { + name string + filledMask string + }{ + { + name: "Fill socket masks", + filledMask: "1111111111111111111111111111111111111111111111111111111111111111", + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask() + mask.Fill() + if mask.String() != string(tc.filledMask) { + t.Errorf("Expected mask to be %v, got %v", tc.filledMask, mask) + } + } +} + +func TestIsEmpty(t *testing.T) { + tcases := []struct { + name string + maskBit int + expectedEmpty bool + }{ + { + name: "Check if mask is empty", + maskBit: int(0), + expectedEmpty: false, + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.maskBit) + empty := mask.IsEmpty() + if empty { + t.Errorf("Expected value to be %v, got %v", tc.expectedEmpty, empty) + } + } +} + +func TestIsSet(t *testing.T) { + tcases := []struct { + name string + maskBit int + expectedSet bool + }{ + { + name: "Check if mask bit is set", + maskBit: int(0), + expectedSet: true, + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.maskBit) + set := mask.IsSet(tc.maskBit) + if !set { + t.Errorf("Expected value to be %v, got %v", tc.expectedSet, set) + } + } +} + +func TestIsEqual(t *testing.T) { + tcases := []struct { + name string + firstMaskBit int + secondMaskBit int + isEqual bool + }{ + { + name: "And socket masks", + firstMaskBit: int(0), + secondMaskBit: int(0), + isEqual: true, + }, + } + for _, tc := range tcases { + firstMask, _ := NewSocketMask(tc.firstMaskBit) + secondMask, _ := NewSocketMask(tc.secondMaskBit) + isEqual := firstMask.IsEqual(secondMask) + if !isEqual { + t.Errorf("Expected mask to be %v, got %v", tc.isEqual, isEqual) + } + } +} + +func TestCount(t *testing.T) { + tcases := []struct { + name string + maskBit int + expectedCount int + }{ + { + name: "Count number of bits set in full mask", + maskBit: 0, + expectedCount: 1, + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.maskBit) + count := mask.Count() + if count != tc.expectedCount { + t.Errorf("Expected value to be %v, got %v", tc.expectedCount, count) + } + } +} + +func TestGetSockets(t *testing.T) { + tcases := []struct { + name string + firstSocket int + secondSocket int + expectedSockets []int + }{ + { + name: "Get number of each socket which has been set", + firstSocket: 0, + secondSocket: 1, + expectedSockets: []int{0, 1}, + }, + } + for _, tc := range tcases { + mask, _ := NewSocketMask(tc.firstSocket, tc.secondSocket) + sockets := mask.GetSockets() + if !reflect.DeepEqual(sockets, tc.expectedSockets) { + t.Errorf("Expected value to be %v, got %v", tc.expectedSockets, sockets) + } + } +} diff --git a/pkg/kubelet/config/defaults.go b/pkg/kubelet/config/defaults.go index 43f7162bfd8..6c1e4ebf411 100644 --- a/pkg/kubelet/config/defaults.go +++ b/pkg/kubelet/config/defaults.go @@ -26,4 +26,5 @@ const ( DefaultKubeletContainersDirName = "containers" DefaultKubeletPluginContainersDirName = "plugin-containers" DefaultKubeletPodResourcesDirName = "pod-resources" + KubeletPluginsDirSELinuxLabel = "system_u:object_r:container_file_t:s0" ) diff --git a/pkg/kubelet/dockershim/docker_container_windows.go b/pkg/kubelet/dockershim/docker_container_windows.go index d14168610f5..4276d7c531f 100644 --- a/pkg/kubelet/dockershim/docker_container_windows.go +++ b/pkg/kubelet/dockershim/docker_container_windows.go @@ -30,7 +30,6 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - "k8s.io/kubernetes/pkg/kubelet/kuberuntime" ) type containerCleanupInfo struct { @@ -54,7 +53,7 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C return cleanupInfo, nil } -// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present, +// applyGMSAConfig looks at the container's .Windows.SecurityContext.GMSACredentialSpec field; if present, // it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value. // We use registry values instead of files since their location cannot change - as opposed to credential spec files, // whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the @@ -63,7 +62,10 @@ func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.C // as it will avoid cluttering the registry - there is a moby PR out for this: // https://github.com/moby/moby/pull/38777 func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCleanupInfo) error { - credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey] + var credSpec string + if config.Windows != nil && config.Windows.SecurityContext != nil { + credSpec = config.Windows.SecurityContext.CredentialSpec + } if credSpec == "" { return nil } diff --git a/pkg/kubelet/dockershim/docker_container_windows_test.go b/pkg/kubelet/dockershim/docker_container_windows_test.go index 2b25ebfbf97..d6c39bcda3f 100644 --- a/pkg/kubelet/dockershim/docker_container_windows_test.go +++ b/pkg/kubelet/dockershim/docker_container_windows_test.go @@ -73,7 +73,11 @@ func TestApplyGMSAConfig(t *testing.T) { expectedValueName := "k8s-cred-spec-" + expectedHex containerConfigWithGMSAAnnotation := &runtimeapi.ContainerConfig{ - Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec}, + Windows: &runtimeapi.WindowsContainerConfig{ + SecurityContext: &runtimeapi.WindowsContainerSecurityContext{ + CredentialSpec: dummyCredSpec, + }, + }, } t.Run("happy path", func(t *testing.T) { diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index 8b156a79299..5d721148bb5 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -96,7 +96,7 @@ func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPod } // Step 2: Create the sandbox container. - if r.GetRuntimeHandler() != "" { + if r.GetRuntimeHandler() != "" && r.GetRuntimeHandler() != runtimeName { return nil, fmt.Errorf("RuntimeHandler %q not supported", r.GetRuntimeHandler()) } createConfig, err := ds.makeSandboxDockerConfig(config, image) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2ac874cf391..65c5eb381b1 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -110,6 +110,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/oom" + "k8s.io/kubernetes/pkg/util/selinux" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/util/subpath" @@ -1047,7 +1048,7 @@ type Kubelet struct { // as it takes time to gather all necessary node information. nodeStatusUpdateFrequency time.Duration - // nodeStatusUpdateFrequency is the frequency that kubelet posts node + // nodeStatusReportFrequency is the frequency that kubelet posts node // status to master. It is only used when node lease feature is enabled. nodeStatusReportFrequency time.Duration @@ -1222,6 +1223,8 @@ type Kubelet struct { // 4. the pod-resources directory func (kl *Kubelet) setupDataDirs() error { kl.rootDirectory = path.Clean(kl.rootDirectory) + pluginRegistrationDir := kl.getPluginsRegistrationDir() + pluginsDir := kl.getPluginsDir() if err := os.MkdirAll(kl.getRootDir(), 0750); err != nil { return fmt.Errorf("error creating root directory: %v", err) } @@ -1240,6 +1243,16 @@ func (kl *Kubelet) setupDataDirs() error { if err := os.MkdirAll(kl.getPodResourcesDir(), 0750); err != nil { return fmt.Errorf("error creating podresources directory: %v", err) } + if selinux.SELinuxEnabled() { + err := selinux.SetFileLabel(pluginRegistrationDir, config.KubeletPluginsDirSELinuxLabel) + if err != nil { + klog.Warningf("Unprivileged containerized plugins might not work. Could not set selinux context on %s: %v", pluginRegistrationDir, err) + } + err = selinux.SetFileLabel(pluginsDir, config.KubeletPluginsDirSELinuxLabel) + if err != nil { + klog.Warningf("Unprivileged containerized plugins might not work. Could not set selinux context on %s: %v", pluginsDir, err) + } + } return nil } @@ -1782,15 +1795,6 @@ func (kl *Kubelet) canRunPod(pod *v1.Pod) lifecycle.PodAdmitResult { } } - // TODO: Refactor as a soft admit handler. - if err := canRunPod(pod); err != nil { - return lifecycle.PodAdmitResult{ - Admit: false, - Reason: "Forbidden", - Message: err.Error(), - } - } - return lifecycle.PodAdmitResult{Admit: true} } @@ -1841,7 +1845,7 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand // 1. configCh: a channel to read config events from // 2. handler: the SyncHandler to dispatch pods to // 3. syncCh: a channel to read periodic sync events from -// 4. houseKeepingCh: a channel to read housekeeping events from +// 4. housekeepingCh: a channel to read housekeeping events from // 5. plegCh: a channel to read PLEG updates from // // Events are also read from the kubelet liveness manager's update channel. @@ -1863,7 +1867,7 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand // handler callback for the event type // * plegCh: update the runtime cache; sync pod // * syncCh: sync all pods waiting for sync -// * houseKeepingCh: trigger cleanup of pods +// * housekeepingCh: trigger cleanup of pods // * liveness manager: sync pods that have failed or in which one or more // containers have failed liveness checks func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handler SyncHandler, diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index ef74e46e72d..94cb68fa2fe 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -159,6 +159,11 @@ func (kl *Kubelet) getPodResourcesDir() string { return filepath.Join(kl.getRootDir(), config.DefaultKubeletPodResourcesDirName) } +// getPluginsDirSELinuxLabel returns the selinux label to be applied on plugin directories +func (kl *Kubelet) getPluginsDirSELinuxLabel() string { + return config.KubeletPluginsDirSELinuxLabel +} + // GetPods returns all pods bound to the kubelet and their spec, and the mirror // pods. func (kl *Kubelet) GetPods() []*v1.Pod { diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index c1d1d499e2b..410b8847bc1 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -26,7 +26,7 @@ import ( "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -132,12 +132,15 @@ func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool { // Zeros out extended resource capacity during reconciliation. func (kl *Kubelet) reconcileExtendedResource(initialNode, node *v1.Node) bool { requiresUpdate := false - for k := range node.Status.Capacity { - if v1helper.IsExtendedResourceName(k) { - klog.Infof("Zero out resource %s capacity in existing node.", k) - node.Status.Capacity[k] = *resource.NewQuantity(int64(0), resource.DecimalSI) - node.Status.Allocatable[k] = *resource.NewQuantity(int64(0), resource.DecimalSI) - requiresUpdate = true + // Check with the device manager to see if node has been recreated, in which case extended resources should be zeroed until they are available + if kl.containerManager.ShouldResetExtendedResourceCapacity() { + for k := range node.Status.Capacity { + if v1helper.IsExtendedResourceName(k) { + klog.Infof("Zero out resource %s capacity in existing node.", k) + node.Status.Capacity[k] = *resource.NewQuantity(int64(0), resource.DecimalSI) + node.Status.Allocatable[k] = *resource.NewQuantity(int64(0), resource.DecimalSI) + requiresUpdate = true + } } } return requiresUpdate diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index 81d51ef2f5e..665f4309d03 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/require" cadvisorapi "github.com/google/cadvisor/info/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -1737,17 +1737,21 @@ func TestUpdateDefaultLabels(t *testing.T) { func TestReconcileExtendedResource(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) testKubelet.kubelet.kubeClient = nil // ensure only the heartbeat client is used + testKubelet.kubelet.containerManager = cm.NewStubContainerManagerWithExtendedResource(true /* shouldResetExtendedResourceCapacity*/) + testKubeletNoReset := newTestKubelet(t, false /* controllerAttachDetachEnabled */) extendedResourceName1 := v1.ResourceName("test.com/resource1") extendedResourceName2 := v1.ResourceName("test.com/resource2") cases := []struct { name string + testKubelet *TestKubelet existingNode *v1.Node expectedNode *v1.Node needsUpdate bool }{ { - name: "no update needed without extended resource", + name: "no update needed without extended resource", + testKubelet: testKubelet, existingNode: &v1.Node{ Status: v1.NodeStatus{ Capacity: v1.ResourceList{ @@ -1779,7 +1783,41 @@ func TestReconcileExtendedResource(t *testing.T) { needsUpdate: false, }, { - name: "extended resource capacity is zeroed", + name: "extended resource capacity is not zeroed due to presence of checkpoint file", + testKubelet: testKubelet, + existingNode: &v1.Node{ + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + Allocatable: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + }, + }, + expectedNode: &v1.Node{ + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + Allocatable: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(5000, resource.BinarySI), + }, + }, + }, + needsUpdate: false, + }, + { + name: "extended resource capacity is zeroed", + testKubelet: testKubeletNoReset, existingNode: &v1.Node{ Status: v1.NodeStatus{ Capacity: v1.ResourceList{ diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index d2d6f89ffba..3eb34fd9884 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -41,7 +41,6 @@ import ( "k8s.io/client-go/util/flowcontrol" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/capabilities" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" @@ -1098,230 +1097,12 @@ func TestGetContainerInfoForMirrorPods(t *testing.T) { require.NotNil(t, stats) } -func TestHostNetworkAllowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostNetworkSources: []string{kubetypes.ApiserverSource, kubetypes.FileSource}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostNetwork: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - kubelet.podManager.SetPods([]*v1.Pod{pod}) - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.NoError(t, err, "expected pod infra creation to succeed") -} - -func TestHostNetworkDisallowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostNetworkSources: []string{}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostNetwork: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.Error(t, err, "expected pod infra creation to fail") -} - -func TestHostPIDAllowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostPIDSources: []string{kubetypes.ApiserverSource, kubetypes.FileSource}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostPID: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - kubelet.podManager.SetPods([]*v1.Pod{pod}) - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.NoError(t, err, "expected pod infra creation to succeed") -} - -func TestHostPIDDisallowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostPIDSources: []string{}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostPID: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.Error(t, err, "expected pod infra creation to fail") -} - -func TestHostIPCAllowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostIPCSources: []string{kubetypes.ApiserverSource, kubetypes.FileSource}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostIPC: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - kubelet.podManager.SetPods([]*v1.Pod{pod}) - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.NoError(t, err, "expected pod infra creation to succeed") -} - -func TestHostIPCDisallowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostIPCSources: []string{}, - }, - }) - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo"}, - }, - HostIPC: true, - }) - pod.Annotations[kubetypes.ConfigSourceAnnotationKey] = kubetypes.FileSource - - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.Error(t, err, "expected pod infra creation to fail") -} - -func TestPrivilegeContainerAllowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - AllowPrivileged: true, - }) - privileged := true - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo", SecurityContext: &v1.SecurityContext{Privileged: &privileged}}, - }, - }) - - kubelet.podManager.SetPods([]*v1.Pod{pod}) - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.NoError(t, err, "expected pod infra creation to succeed") -} - -func TestPrivilegedContainerDisallowed(t *testing.T) { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - kubelet := testKubelet.kubelet - - capabilities.SetForTests(capabilities.Capabilities{ - AllowPrivileged: false, - }) - privileged := true - pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ - Containers: []v1.Container{ - {Name: "foo", SecurityContext: &v1.SecurityContext{Privileged: &privileged}}, - }, - }) - - err := kubelet.syncPod(syncPodOptions{ - pod: pod, - podStatus: &kubecontainer.PodStatus{}, - updateType: kubetypes.SyncPodUpdate, - }) - assert.Error(t, err, "expected pod infra creation to fail") -} - func TestNetworkErrorsWithoutHostNetwork(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.runtimeState.setNetworkState(fmt.Errorf("simulated network error")) - capabilities.SetForTests(capabilities.Capabilities{ - PrivilegedSources: capabilities.PrivilegedSources{ - HostNetworkSources: []string{kubetypes.ApiserverSource, kubetypes.FileSource}, - }, - }) pod := podWithUIDNameNsSpec("12345678", "hostnetwork", "new", v1.PodSpec{ HostNetwork: false, diff --git a/pkg/kubelet/kuberuntime/BUILD b/pkg/kubelet/kuberuntime/BUILD index efdbafb2e1a..4e10569e8c7 100644 --- a/pkg/kubelet/kuberuntime/BUILD +++ b/pkg/kubelet/kuberuntime/BUILD @@ -90,7 +90,6 @@ go_test( "instrumented_services_test.go", "kuberuntime_container_linux_test.go", "kuberuntime_container_test.go", - "kuberuntime_container_windows_test.go", "kuberuntime_gc_test.go", "kuberuntime_image_test.go", "kuberuntime_manager_test.go", diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 32590604c40..2983bbec76a 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -194,7 +194,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai return nil, nil, err } - uid, username, err := m.getImageUser(imageRef) + uid, username, err := m.getImageUser(container.Image) if err != nil { return nil, cleanupAction, err } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go index db03eae34e3..e99660c3c2f 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_windows.go @@ -36,12 +36,8 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config if err != nil { return err } - - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) { - determineEffectiveSecurityContext(config, container, pod) - } - config.Windows = windowsConfig + return nil } @@ -100,43 +96,11 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1 if username != "" { wc.SecurityContext.RunAsUsername = username } + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) && + effectiveSc.WindowsOptions != nil && + effectiveSc.WindowsOptions.GMSACredentialSpec != nil { + wc.SecurityContext.CredentialSpec = *effectiveSc.WindowsOptions.GMSACredentialSpec + } return wc, nil } - -const ( - // GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use. - GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec" - // gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller - // stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name - // with this suffix appended). - gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey - // gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA - // credential spec to use for containers that do not have their own specific GMSA cred spec set via a - // gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above - gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec" -) - -// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's -// GMSASpecContainerAnnotationKey annotation. -func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) { - var containerCredSpec string - - containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix - if pod.Annotations[containerGMSAPodAnnotation] != "" { - containerCredSpec = pod.Annotations[containerGMSAPodAnnotation] - } else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" { - containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey] - } - - if containerCredSpec != "" { - if config.Annotations == nil { - config.Annotations = make(map[string]string) - } - config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec - } else { - // the annotation shouldn't be present, but let's err on the side of caution: - // it should only be set here and nowhere else - delete(config.Annotations, GMSASpecContainerAnnotationKey) - } -} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go deleted file mode 100644 index f4ea8e4dd5d..00000000000 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kuberuntime - -import ( - "github.com/stretchr/testify/assert" - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" -) - -func TestDetermineEffectiveSecurityContext(t *testing.T) { - containerName := "container_name" - container := &corev1.Container{Name: containerName} - dummyCredSpec := "test cred spec contents" - - buildPod := func(annotations map[string]string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: annotations, - }, - } - } - - t.Run("when there's a specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) { - containerConfig := &runtimeapi.ContainerConfig{} - - pod := buildPod(map[string]string{ - "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, - }) - - determineEffectiveSecurityContext(containerConfig, container, pod) - - assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) - }) - t.Run("when there's a specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) { - containerConfig := &runtimeapi.ContainerConfig{} - - pod := buildPod(map[string]string{ - "container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, - "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": "should be ignored", - }) - - determineEffectiveSecurityContext(containerConfig, container, pod) - - assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) - }) - t.Run("when there's no specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) { - containerConfig := &runtimeapi.ContainerConfig{} - - pod := buildPod(map[string]string{ - "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec, - }) - - determineEffectiveSecurityContext(containerConfig, container, pod) - - assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"]) - }) - t.Run("when there's no specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) { - containerConfig := &runtimeapi.ContainerConfig{} - - determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{}) - - assert.Nil(t, containerConfig.Annotations) - }) -} diff --git a/pkg/kubelet/util.go b/pkg/kubelet/util.go index 3047c2fa619..bdf22af64e0 100644 --- a/pkg/kubelet/util.go +++ b/pkg/kubelet/util.go @@ -17,105 +17,9 @@ limitations under the License. package kubelet import ( - "fmt" "os" - - "k8s.io/api/core/v1" - "k8s.io/kubernetes/pkg/capabilities" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/kubernetes/pkg/securitycontext" ) -// Check whether we have the capabilities to run the specified pod. -func canRunPod(pod *v1.Pod) error { - if !capabilities.Get().AllowPrivileged { - for _, container := range pod.Spec.Containers { - if securitycontext.HasPrivilegedRequest(&container) { - return fmt.Errorf("pod with UID %q specified privileged container, but is disallowed", pod.UID) - } - } - for _, container := range pod.Spec.InitContainers { - if securitycontext.HasPrivilegedRequest(&container) { - return fmt.Errorf("pod with UID %q specified privileged init container, but is disallowed", pod.UID) - } - } - } - - if pod.Spec.HostNetwork { - allowed, err := allowHostNetwork(pod) - if err != nil { - return err - } - if !allowed { - return fmt.Errorf("pod with UID %q specified host networking, but is disallowed", pod.UID) - } - } - - if pod.Spec.HostPID { - allowed, err := allowHostPID(pod) - if err != nil { - return err - } - if !allowed { - return fmt.Errorf("pod with UID %q specified host PID, but is disallowed", pod.UID) - } - } - - if pod.Spec.HostIPC { - allowed, err := allowHostIPC(pod) - if err != nil { - return err - } - if !allowed { - return fmt.Errorf("pod with UID %q specified host ipc, but is disallowed", pod.UID) - } - } - - return nil -} - -// Determined whether the specified pod is allowed to use host networking -func allowHostNetwork(pod *v1.Pod) (bool, error) { - podSource, err := kubetypes.GetPodSource(pod) - if err != nil { - return false, err - } - for _, source := range capabilities.Get().PrivilegedSources.HostNetworkSources { - if source == podSource { - return true, nil - } - } - return false, nil -} - -// Determined whether the specified pod is allowed to use host PID -func allowHostPID(pod *v1.Pod) (bool, error) { - podSource, err := kubetypes.GetPodSource(pod) - if err != nil { - return false, err - } - for _, source := range capabilities.Get().PrivilegedSources.HostPIDSources { - if source == podSource { - return true, nil - } - } - return false, nil -} - -// Determined whether the specified pod is allowed to use host ipc -func allowHostIPC(pod *v1.Pod) (bool, error) { - podSource, err := kubetypes.GetPodSource(pod) - if err != nil { - return false, err - } - for _, source := range capabilities.Get().PrivilegedSources.HostIPCSources { - if source == podSource { - return true, nil - } - } - return false, nil -} - // dirExists returns true if the path exists and represents a directory. func dirExists(path string) bool { s, err := os.Stat(path) diff --git a/pkg/proxy/ipvs/BUILD b/pkg/proxy/ipvs/BUILD index 667eacea905..39b2ce85406 100644 --- a/pkg/proxy/ipvs/BUILD +++ b/pkg/proxy/ipvs/BUILD @@ -9,6 +9,7 @@ load( go_test( name = "go_default_test", srcs = [ + "graceful_termination_test.go", "ipset_test.go", "proxier_test.go", ], diff --git a/pkg/proxy/ipvs/graceful_termination_test.go b/pkg/proxy/ipvs/graceful_termination_test.go new file mode 100644 index 00000000000..c99feef993f --- /dev/null +++ b/pkg/proxy/ipvs/graceful_termination_test.go @@ -0,0 +1,338 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipvs + +import ( + "net" + "reflect" + "testing" + + utilipvs "k8s.io/kubernetes/pkg/util/ipvs" + utilipvstest "k8s.io/kubernetes/pkg/util/ipvs/testing" +) + +func Test_GracefulDeleteRS(t *testing.T) { + tests := []struct { + name string + vs *utilipvs.VirtualServer + rs *utilipvs.RealServer + existingIPVS *utilipvstest.FakeIPVS + expectedIPVS *utilipvstest.FakeIPVS + err error + }{ + { + name: "graceful delete, no connections results in deleting the real server immediatetly", + vs: &utilipvs.VirtualServer{ + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + rs: &utilipvs.RealServer{ + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 0, + }, + existingIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 0, + }, + }, + }, + }, + expectedIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: {}, + }, + }, + err: nil, + }, + { + name: "graceful delete, real server has active connections, weight should be 0 but don't delete", + vs: &utilipvs.VirtualServer{ + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + rs: &utilipvs.RealServer{ + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 10, + InactiveConn: 0, + }, + existingIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 10, + InactiveConn: 0, + }, + }, + }, + }, + expectedIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 0, + ActiveConn: 10, + InactiveConn: 0, + }, + }, + }, + }, + err: nil, + }, + { + name: "graceful delete, real server has in-active connections, weight should be 0 but don't delete", + vs: &utilipvs.VirtualServer{ + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + rs: &utilipvs.RealServer{ + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 10, + }, + existingIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 10, + }, + }, + }, + }, + expectedIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 0, + ActiveConn: 0, + InactiveConn: 10, + }, + }, + }, + }, + err: nil, + }, + { + name: "graceful delete, real server mismatch should be no-op", + vs: &utilipvs.VirtualServer{ + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + rs: &utilipvs.RealServer{ + Address: net.ParseIP("10.0.0.1"), + Port: uint16(81), // port mismatched + Weight: 100, + ActiveConn: 0, + InactiveConn: 10, + }, + existingIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 10, + }, + }, + }, + }, + expectedIPVS: &utilipvstest.FakeIPVS{ + Services: map[utilipvstest.ServiceKey]*utilipvs.VirtualServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + Address: net.ParseIP("1.1.1.1"), + Protocol: "tcp", + Port: uint16(80), + }, + }, + Destinations: map[utilipvstest.ServiceKey][]*utilipvs.RealServer{ + { + IP: "1.1.1.1", + Port: 80, + Protocol: "tcp", + }: { + { + Address: net.ParseIP("10.0.0.1"), + Port: uint16(80), + Weight: 100, + ActiveConn: 0, + InactiveConn: 10, + }, + }, + }, + }, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ipvs := test.existingIPVS + gracefulTerminationManager := NewGracefulTerminationManager(ipvs) + + err := gracefulTerminationManager.GracefulDeleteRS(test.vs, test.rs) + if err != test.err { + t.Logf("actual err: %v", err) + t.Logf("expected err: %v", test.err) + t.Errorf("unexpected error") + } + + if !reflect.DeepEqual(ipvs, test.expectedIPVS) { + t.Logf("actual: %+v", ipvs) + t.Logf("expected : %+v", test.expectedIPVS) + t.Errorf("unexpected IPVS servers") + } + }) + } +} diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go index e449cc1ee3f..636c7c95282 100644 --- a/pkg/proxy/ipvs/ipset.go +++ b/pkg/proxy/ipvs/ipset.go @@ -65,11 +65,21 @@ const ( kubeNodePortLocalSetUDPComment = "Kubernetes nodeport UDP port with externalTrafficPolicy=local" kubeNodePortLocalSetUDP = "KUBE-NODE-PORT-LOCAL-UDP" - kubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose" - kubeNodePortSetSCTP = "KUBE-NODE-PORT-SCTP" + // This ipset is no longer active but still used in previous versions. + // DO NOT create an ipset using this name + legacyKubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose" + legacyKubeNodePortSetSCTP = "KUBE-NODE-PORT-SCTP" - kubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local" - kubeNodePortLocalSetSCTP = "KUBE-NODE-PORT-LOCAL-SCTP" + // This ipset is no longer active but still used in previous versions. + // DO NOT create an ipset using this name + legacyKubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local" + legacyKubeNodePortLocalSetSCTP = "KUBE-NODE-PORT-LOCAL-SCTP" + + kubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose with type 'hash ip:port'" + kubeNodePortSetSCTP = "KUBE-NODE-PORT-SCTP-HASH" + + kubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local with type 'hash ip:port'" + kubeNodePortLocalSetSCTP = "KUBE-NODE-PORT-LOCAL-SCTP-HASH" ) // IPSetVersioner can query the current ipset version. diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 288dadc5476..2e46f18435e 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -107,24 +107,6 @@ func (fake *fakeIPSetVersioner) GetVersion() (string, error) { return fake.version, fake.err } -// New returns a new FakeSysctl -func NewFakeSysctl() *FakeSysctl { - return &FakeSysctl{} -} - -type FakeSysctl struct { -} - -// GetSysctl returns the value for the specified sysctl setting -func (fakeSysctl *FakeSysctl) GetSysctl(sysctl string) (int, error) { - return 1, nil -} - -// SetSysctl modifies the specified sysctl flag to the new value -func (fakeSysctl *FakeSysctl) SetSysctl(sysctl string, newVal int) error { - return nil -} - func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP, excludeCIDRs []*net.IPNet) *Proxier { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index 73b4646edc4..e0496df8dbd 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -387,19 +387,26 @@ func (c *configFactory) CreateFromConfig(policy schedulerapi.Policy) (*Config, e var extenders []algorithm.SchedulerExtender if len(policy.ExtenderConfigs) != 0 { ignoredExtendedResources := sets.NewString() + var ignorableExtenders []algorithm.SchedulerExtender for ii := range policy.ExtenderConfigs { klog.V(2).Infof("Creating extender with config %+v", policy.ExtenderConfigs[ii]) extender, err := core.NewHTTPExtender(&policy.ExtenderConfigs[ii]) if err != nil { return nil, err } - extenders = append(extenders, extender) + if !extender.IsIgnorable() { + extenders = append(extenders, extender) + } else { + ignorableExtenders = append(ignorableExtenders, extender) + } for _, r := range policy.ExtenderConfigs[ii].ManagedResources { if r.IgnoredByScheduler { ignoredExtendedResources.Insert(string(r.Name)) } } } + // place ignorable extenders to the tail of extenders + extenders = append(extenders, ignorableExtenders...) predicates.RegisterPredicateMetadataProducerWithExtendedResourceOptions(ignoredExtendedResources) } // Providing HardPodAffinitySymmetricWeight in the policy config is the new and preferred way of providing the value. diff --git a/pkg/scheduler/framework/v1alpha1/interface.go b/pkg/scheduler/framework/v1alpha1/interface.go index 09866b9d723..1a47f3f9226 100644 --- a/pkg/scheduler/framework/v1alpha1/interface.go +++ b/pkg/scheduler/framework/v1alpha1/interface.go @@ -34,14 +34,14 @@ type Code int const ( // Success means that plugin ran correctly and found pod schedulable. // NOTE: A nil status is also considered as "Success". - Success Code = 0 + Success Code = iota // Error is used for internal plugin errors, unexpected input, etc. - Error Code = 1 + Error // Unschedulable is used when a plugin finds a pod unschedulable. // The accompanying status message should explain why the pod is unschedulable. - Unschedulable Code = 2 + Unschedulable // Wait is used when a permit plugin finds a pod scheduling should wait. - Wait Code = 3 + Wait ) // Status indicates the result of running a plugin. It consists of a code and a diff --git a/pkg/securitycontext/util.go b/pkg/securitycontext/util.go index a39ee7571a8..6a012076dc9 100644 --- a/pkg/securitycontext/util.go +++ b/pkg/securitycontext/util.go @@ -66,6 +66,18 @@ func DetermineEffectiveSecurityContext(pod *v1.Pod, container *v1.Container) *v1 *effectiveSc.SELinuxOptions = *containerSc.SELinuxOptions } + if containerSc.WindowsOptions != nil { + // only override fields that are set at the container level, not the whole thing + if effectiveSc.WindowsOptions == nil { + effectiveSc.WindowsOptions = &v1.WindowsSecurityContextOptions{} + } + if containerSc.WindowsOptions.GMSACredentialSpecName != nil || containerSc.WindowsOptions.GMSACredentialSpec != nil { + // both GMSA fields go hand in hand + effectiveSc.WindowsOptions.GMSACredentialSpecName = containerSc.WindowsOptions.GMSACredentialSpecName + effectiveSc.WindowsOptions.GMSACredentialSpec = containerSc.WindowsOptions.GMSACredentialSpec + } + } + if containerSc.Capabilities != nil { effectiveSc.Capabilities = new(v1.Capabilities) *effectiveSc.Capabilities = *containerSc.Capabilities @@ -120,6 +132,12 @@ func securityContextFromPodSecurityContext(pod *v1.Pod) *v1.SecurityContext { synthesized.SELinuxOptions = &v1.SELinuxOptions{} *synthesized.SELinuxOptions = *pod.Spec.SecurityContext.SELinuxOptions } + + if pod.Spec.SecurityContext.WindowsOptions != nil { + synthesized.WindowsOptions = &v1.WindowsSecurityContextOptions{} + *synthesized.WindowsOptions = *pod.Spec.SecurityContext.WindowsOptions + } + if pod.Spec.SecurityContext.RunAsUser != nil { synthesized.RunAsUser = new(int64) *synthesized.RunAsUser = *pod.Spec.SecurityContext.RunAsUser diff --git a/pkg/util/ipvs/testing/fake.go b/pkg/util/ipvs/testing/fake.go index bfc854bb365..14d16e893d7 100644 --- a/pkg/util/ipvs/testing/fake.go +++ b/pkg/util/ipvs/testing/fake.go @@ -27,47 +27,49 @@ import ( //FakeIPVS no-op implementation of ipvs Interface type FakeIPVS struct { Scheduler string - Services map[serviceKey]*utilipvs.VirtualServer - Destinations map[serviceKey][]*utilipvs.RealServer + Services map[ServiceKey]*utilipvs.VirtualServer + Destinations map[ServiceKey][]*utilipvs.RealServer } -type serviceKey struct { +// ServiceKey uniquely identifies a Service for an IPVS virtual server +type ServiceKey struct { IP string Port uint16 Protocol string } -func (s *serviceKey) String() string { +func (s *ServiceKey) String() string { return fmt.Sprintf("%s:%d/%s", s.IP, s.Port, s.Protocol) } -type realServerKey struct { +// RealServerKey uniquely identifies an Endpoint for an IPVS real server +type RealServerKey struct { Address net.IP Port uint16 } -func (r *realServerKey) String() string { +func (r *RealServerKey) String() string { return net.JoinHostPort(r.Address.String(), strconv.Itoa(int(r.Port))) } //NewFake creates a fake ipvs implementation - a cache store. func NewFake() *FakeIPVS { return &FakeIPVS{ - Services: make(map[serviceKey]*utilipvs.VirtualServer), - Destinations: make(map[serviceKey][]*utilipvs.RealServer), + Services: make(map[ServiceKey]*utilipvs.VirtualServer), + Destinations: make(map[ServiceKey][]*utilipvs.RealServer), } } -func toServiceKey(serv *utilipvs.VirtualServer) serviceKey { - return serviceKey{ +func toServiceKey(serv *utilipvs.VirtualServer) ServiceKey { + return ServiceKey{ IP: serv.Address.String(), Port: serv.Port, Protocol: serv.Protocol, } } -func toRealServerKey(rs *utilipvs.RealServer) *realServerKey { - return &realServerKey{ +func toRealServerKey(rs *utilipvs.RealServer) *RealServerKey { + return &RealServerKey{ Address: rs.Address, Port: rs.Port, } diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index cc313909fc7..96dc68c9a03 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -53,7 +53,7 @@ type Interface interface { // is a mountpoint. // It should return ErrNotExist when the directory does not exist. // IsLikelyNotMountPoint does NOT properly detect all mountpoint types - // most notably linux bind mounts. + // most notably linux bind mounts and symbolic link. IsLikelyNotMountPoint(file string) (bool, error) // DeviceOpened determines if the device is in use elsewhere // on the system, i.e. still mounted. diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 71ac88c2eb5..0f75d2184d5 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -228,6 +228,7 @@ func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { // IsLikelyNotMountPoint determines if a directory is not a mountpoint. // It is fast but not necessarily ALWAYS correct. If the path is in fact // a bind mount from one part of a mount to another it will not be detected. +// It also can not distinguish between mountpoints and symbolic links. // mkdir /tmp/a /tmp/b; mount --bind /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") // will return true. When in fact /tmp/b is a mount point. If this situation // if of interest to you, don't use this function... diff --git a/pkg/util/selinux/selinux_linux.go b/pkg/util/selinux/selinux_linux.go index bdaa505c5d5..33ae35884fe 100644 --- a/pkg/util/selinux/selinux_linux.go +++ b/pkg/util/selinux/selinux_linux.go @@ -50,3 +50,8 @@ func (_ *realSELinuxRunner) Getfilecon(path string) (string, error) { } return selinux.FileLabel(path) } + +// SetFileLabel applies the SELinux label on the path or returns an error. +func SetFileLabel(path string, label string) error { + return selinux.SetFileLabel(path, label) +} diff --git a/pkg/util/selinux/selinux_unsupported.go b/pkg/util/selinux/selinux_unsupported.go index 4f7767472c9..4c8f5f0b28c 100644 --- a/pkg/util/selinux/selinux_unsupported.go +++ b/pkg/util/selinux/selinux_unsupported.go @@ -31,3 +31,8 @@ var _ SELinuxRunner = &realSELinuxRunner{} func (_ *realSELinuxRunner) Getfilecon(path string) (string, error) { return "", nil } + +// FileLabel returns the SELinux label for this path or returns an error. +func SetFileLabel(path string, label string) error { + return nil +} diff --git a/pkg/volume/azure_dd/BUILD b/pkg/volume/azure_dd/BUILD index 4833f5344cc..11a22977569 100644 --- a/pkg/volume/azure_dd/BUILD +++ b/pkg/volume/azure_dd/BUILD @@ -16,6 +16,7 @@ go_library( "azure_common_windows.go", "azure_dd.go", "azure_dd_block.go", + "azure_dd_max_disk_count.go", "azure_mounter.go", "azure_provision.go", ], @@ -76,7 +77,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute:go_default_library", - "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 3df26078f92..613408955fb 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "strings" - "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage" @@ -87,9 +86,6 @@ var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{} var _ volume.ExpandableVolumePlugin = &azureDataDiskPlugin{} var _ volume.DeviceMountableVolumePlugin = &azureDataDiskPlugin{} -// store vm size list in current region -var vmSizeList *[]compute.VirtualMachineSize - const ( azureDataDiskPluginName = "kubernetes.io/azure-disk" defaultAzureVolumeLimit = 16 @@ -164,40 +160,22 @@ func (plugin *azureDataDiskPlugin) GetVolumeLimits() (map[string]int64, error) { return volumeLimits, nil } - if vmSizeList == nil { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - result, err := az.VirtualMachineSizesClient.List(ctx, az.Location) - if err != nil || result.Value == nil { - klog.Errorf("failed to list vm sizes in GetVolumeLimits, plugin.host: %s, location: %s", plugin.host.GetHostName(), az.Location) - return volumeLimits, nil - } - vmSizeList = result.Value - } - volumeLimits = map[string]int64{ - util.AzureVolumeLimitKey: getMaxDataDiskCount(instanceType, vmSizeList), + util.AzureVolumeLimitKey: getMaxDataDiskCount(instanceType), } return volumeLimits, nil } -func getMaxDataDiskCount(instanceType string, sizeList *[]compute.VirtualMachineSize) int64 { - if sizeList == nil { - return defaultAzureVolumeLimit +func getMaxDataDiskCount(instanceType string) int64 { + vmsize := strings.ToUpper(instanceType) + maxDataDiskCount, exists := maxDataDiskCountMap[vmsize] + if exists { + klog.V(12).Infof("got a matching size in getMaxDataDiskCount, VM Size: %s, MaxDataDiskCount: %d", vmsize, maxDataDiskCount) + return maxDataDiskCount } - vmsize := strings.ToUpper(instanceType) - for _, size := range *sizeList { - if size.Name == nil || size.MaxDataDiskCount == nil { - klog.Errorf("failed to get vm size in getMaxDataDiskCount") - continue - } - if strings.ToUpper(*size.Name) == vmsize { - klog.V(12).Infof("got a matching size in getMaxDataDiskCount, Name: %s, MaxDataDiskCount: %d", *size.Name, *size.MaxDataDiskCount) - return int64(*size.MaxDataDiskCount) - } - } + klog.V(12).Infof("not found a matching size in getMaxDataDiskCount, VM Size: %s, use default volume limit: %d", vmsize, defaultAzureVolumeLimit) return defaultAzureVolumeLimit } diff --git a/pkg/volume/azure_dd/azure_dd_max_disk_count.go b/pkg/volume/azure_dd/azure_dd_max_disk_count.go new file mode 100644 index 00000000000..a5dc0d44028 --- /dev/null +++ b/pkg/volume/azure_dd/azure_dd_max_disk_count.go @@ -0,0 +1,264 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_dd + +// about how to get all VM size list, +// refer to https://github.com/kubernetes/kubernetes/issues/77461#issuecomment-492488756 +var maxDataDiskCountMap = map[string]int64{ + "BASIC_A0": 1, + "BASIC_A1": 2, + "BASIC_A2": 4, + "BASIC_A3": 8, + "BASIC_A4": 16, + "STANDARD_A0": 1, + "STANDARD_A10": 32, + "STANDARD_A11": 64, + "STANDARD_A1": 2, + "STANDARD_A1_V2": 2, + "STANDARD_A2": 4, + "STANDARD_A2M_V2": 4, + "STANDARD_A2_V2": 4, + "STANDARD_A3": 8, + "STANDARD_A4": 16, + "STANDARD_A4M_V2": 8, + "STANDARD_A4_V2": 8, + "STANDARD_A5": 4, + "STANDARD_A6": 8, + "STANDARD_A7": 16, + "STANDARD_A8": 32, + "STANDARD_A8M_V2": 16, + "STANDARD_A8_V2": 16, + "STANDARD_A9": 64, + "STANDARD_B1LS": 2, + "STANDARD_B1MS": 2, + "STANDARD_B1S": 2, + "STANDARD_B2MS": 4, + "STANDARD_B2S": 4, + "STANDARD_B4MS": 8, + "STANDARD_B8MS": 16, + "STANDARD_D11": 8, + "STANDARD_D11_V2": 8, + "STANDARD_D11_V2_PROMO": 8, + "STANDARD_D12": 16, + "STANDARD_D12_V2": 16, + "STANDARD_D12_V2_PROMO": 16, + "STANDARD_D13": 32, + "STANDARD_D13_V2": 32, + "STANDARD_D13_V2_PROMO": 32, + "STANDARD_D1": 4, + "STANDARD_D14": 64, + "STANDARD_D14_V2": 64, + "STANDARD_D14_V2_PROMO": 64, + "STANDARD_D15_V2": 64, + "STANDARD_D16S_V3": 32, + "STANDARD_D16_V3": 32, + "STANDARD_D1_V2": 4, + "STANDARD_D2": 8, + "STANDARD_D2S_V3": 4, + "STANDARD_D2_V2": 8, + "STANDARD_D2_V2_PROMO": 8, + "STANDARD_D2_V3": 4, + "STANDARD_D3": 16, + "STANDARD_D32S_V3": 32, + "STANDARD_D32_V3": 32, + "STANDARD_D3_V2": 16, + "STANDARD_D3_V2_PROMO": 16, + "STANDARD_D4": 32, + "STANDARD_D4S_V3": 8, + "STANDARD_D4_V2": 32, + "STANDARD_D4_V2_PROMO": 32, + "STANDARD_D4_V3": 8, + "STANDARD_D5_V2": 64, + "STANDARD_D5_V2_PROMO": 64, + "STANDARD_D64S_V3": 32, + "STANDARD_D64_V3": 32, + "STANDARD_D8S_V3": 16, + "STANDARD_D8_V3": 16, + "STANDARD_DC2S": 2, + "STANDARD_DC4S": 4, + "STANDARD_DS11-1_V2": 8, + "STANDARD_DS11": 8, + "STANDARD_DS11_V2": 8, + "STANDARD_DS11_V2_PROMO": 8, + "STANDARD_DS12": 16, + "STANDARD_DS12-1_V2": 16, + "STANDARD_DS12-2_V2": 16, + "STANDARD_DS12_V2": 16, + "STANDARD_DS12_V2_PROMO": 16, + "STANDARD_DS13-2_V2": 32, + "STANDARD_DS13": 32, + "STANDARD_DS13-4_V2": 32, + "STANDARD_DS13_V2": 32, + "STANDARD_DS13_V2_PROMO": 32, + "STANDARD_DS1": 4, + "STANDARD_DS14-4_V2": 64, + "STANDARD_DS14": 64, + "STANDARD_DS14-8_V2": 64, + "STANDARD_DS14_V2": 64, + "STANDARD_DS14_V2_PROMO": 64, + "STANDARD_DS15_V2": 64, + "STANDARD_DS1_V2": 4, + "STANDARD_DS2": 8, + "STANDARD_DS2_V2": 8, + "STANDARD_DS2_V2_PROMO": 8, + "STANDARD_DS3": 16, + "STANDARD_DS3_V2": 16, + "STANDARD_DS3_V2_PROMO": 16, + "STANDARD_DS4": 32, + "STANDARD_DS4_V2": 32, + "STANDARD_DS4_V2_PROMO": 32, + "STANDARD_DS5_V2": 64, + "STANDARD_DS5_V2_PROMO": 64, + "STANDARD_E16-4S_V3": 32, + "STANDARD_E16-8S_V3": 32, + "STANDARD_E16S_V3": 32, + "STANDARD_E16_V3": 32, + "STANDARD_E20S_V3": 32, + "STANDARD_E20_V3": 32, + "STANDARD_E2S_V3": 4, + "STANDARD_E2_V3": 4, + "STANDARD_E32-16S_V3": 32, + "STANDARD_E32-8S_V3": 32, + "STANDARD_E32S_V3": 32, + "STANDARD_E32_V3": 32, + "STANDARD_E4-2S_V3": 8, + "STANDARD_E4S_V3": 8, + "STANDARD_E4_V3": 8, + "STANDARD_E64-16S_V3": 32, + "STANDARD_E64-32S_V3": 32, + "STANDARD_E64IS_V3": 32, + "STANDARD_E64I_V3": 32, + "STANDARD_E64S_V3": 32, + "STANDARD_E64_V3": 32, + "STANDARD_E8-2S_V3": 16, + "STANDARD_E8-4S_V3": 16, + "STANDARD_E8S_V3": 16, + "STANDARD_E8_V3": 16, + "STANDARD_F1": 4, + "STANDARD_F16": 64, + "STANDARD_F16S": 64, + "STANDARD_F16S_V2": 32, + "STANDARD_F1S": 4, + "STANDARD_F2": 8, + "STANDARD_F2S": 8, + "STANDARD_F2S_V2": 4, + "STANDARD_F32S_V2": 32, + "STANDARD_F4": 16, + "STANDARD_F4S": 16, + "STANDARD_F4S_V2": 8, + "STANDARD_F64S_V2": 32, + "STANDARD_F72S_V2": 32, + "STANDARD_F8": 32, + "STANDARD_F8S": 32, + "STANDARD_F8S_V2": 16, + "STANDARD_G1": 8, + "STANDARD_G2": 16, + "STANDARD_G3": 32, + "STANDARD_G4": 64, + "STANDARD_G5": 64, + "STANDARD_GS1": 8, + "STANDARD_GS2": 16, + "STANDARD_GS3": 32, + "STANDARD_GS4-4": 64, + "STANDARD_GS4": 64, + "STANDARD_GS4-8": 64, + "STANDARD_GS5-16": 64, + "STANDARD_GS5": 64, + "STANDARD_GS5-8": 64, + "STANDARD_H16": 64, + "STANDARD_H16M": 64, + "STANDARD_H16M_PROMO": 64, + "STANDARD_H16MR": 64, + "STANDARD_H16MR_PROMO": 64, + "STANDARD_H16_PROMO": 64, + "STANDARD_H16R": 64, + "STANDARD_H16R_PROMO": 64, + "STANDARD_H8": 32, + "STANDARD_H8M": 32, + "STANDARD_H8M_PROMO": 32, + "STANDARD_H8_PROMO": 32, + "STANDARD_HB60RS": 4, + "STANDARD_HC44RS": 4, + "STANDARD_L16S": 64, + "STANDARD_L16S_V2": 32, + "STANDARD_L32S": 64, + "STANDARD_L32S_V2": 32, + "STANDARD_L4S": 16, + "STANDARD_L64S_V2": 32, + "STANDARD_L80S_V2": 32, + "STANDARD_L8S": 32, + "STANDARD_L8S_V2": 16, + "STANDARD_M128-32MS": 64, + "STANDARD_M128": 64, + "STANDARD_M128-64MS": 64, + "STANDARD_M128M": 64, + "STANDARD_M128MS": 64, + "STANDARD_M128S": 64, + "STANDARD_M16-4MS": 16, + "STANDARD_M16-8MS": 16, + "STANDARD_M16MS": 16, + "STANDARD_M208MS_V2": 64, + "STANDARD_M208S_V2": 64, + "STANDARD_M32-16MS": 32, + "STANDARD_M32-8MS": 32, + "STANDARD_M32LS": 32, + "STANDARD_M32MS": 32, + "STANDARD_M32TS": 32, + "STANDARD_M64-16MS": 64, + "STANDARD_M64-32MS": 64, + "STANDARD_M64": 64, + "STANDARD_M64LS": 64, + "STANDARD_M64M": 64, + "STANDARD_M64MS": 64, + "STANDARD_M64S": 64, + "STANDARD_M8-2MS": 8, + "STANDARD_M8-4MS": 8, + "STANDARD_M8MS": 8, + "STANDARD_NC12": 48, + "STANDARD_NC12_PROMO": 48, + "STANDARD_NC12S_V2": 24, + "STANDARD_NC12S_V3": 24, + "STANDARD_NC24": 64, + "STANDARD_NC24_PROMO": 64, + "STANDARD_NC24R": 64, + "STANDARD_NC24R_PROMO": 64, + "STANDARD_NC24RS_V2": 32, + "STANDARD_NC24RS_V3": 32, + "STANDARD_NC24S_V2": 32, + "STANDARD_NC24S_V3": 32, + "STANDARD_NC6": 24, + "STANDARD_NC6_PROMO": 24, + "STANDARD_NC6S_V2": 12, + "STANDARD_NC6S_V3": 12, + "STANDARD_ND12S": 24, + "STANDARD_ND24RS": 32, + "STANDARD_ND24S": 32, + "STANDARD_ND6S": 12, + "STANDARD_NV12": 48, + "STANDARD_NV12_PROMO": 48, + "STANDARD_NV12S_V2": 24, + "STANDARD_NV12S_V3": 12, + "STANDARD_NV24": 64, + "STANDARD_NV24_PROMO": 64, + "STANDARD_NV24S_V2": 32, + "STANDARD_NV24S_V3": 24, + "STANDARD_NV48S_V3": 32, + "STANDARD_NV6": 24, + "STANDARD_NV6_PROMO": 24, + "STANDARD_NV6S_V2": 12, + "STANDARD_PB6S": 12, +} diff --git a/pkg/volume/azure_dd/azure_dd_test.go b/pkg/volume/azure_dd/azure_dd_test.go index dfe922872e1..82fa41bc53c 100644 --- a/pkg/volume/azure_dd/azure_dd_test.go +++ b/pkg/volume/azure_dd/azure_dd_test.go @@ -20,8 +20,6 @@ import ( "os" "testing" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute" - "github.com/Azure/go-autorest/autorest/to" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" @@ -61,33 +59,24 @@ func TestCanSupport(t *testing.T) { func TestGetMaxDataDiskCount(t *testing.T) { tests := []struct { instanceType string - sizeList *[]compute.VirtualMachineSize expectResult int64 }{ { instanceType: "standard_d2_v2", - sizeList: &[]compute.VirtualMachineSize{ - {Name: to.StringPtr("Standard_D2_V2"), MaxDataDiskCount: to.Int32Ptr(8)}, - {Name: to.StringPtr("Standard_D3_V2"), MaxDataDiskCount: to.Int32Ptr(16)}, - }, expectResult: 8, }, { instanceType: "NOT_EXISTING", - sizeList: &[]compute.VirtualMachineSize{ - {Name: to.StringPtr("Standard_D2_V2"), MaxDataDiskCount: to.Int32Ptr(8)}, - }, expectResult: defaultAzureVolumeLimit, }, { instanceType: "", - sizeList: &[]compute.VirtualMachineSize{}, expectResult: defaultAzureVolumeLimit, }, } for _, test := range tests { - result := getMaxDataDiskCount(test.instanceType, test.sizeList) + result := getMaxDataDiskCount(test.instanceType) assert.Equal(t, test.expectResult, result) } } diff --git a/pkg/volume/csi/csi_attacher_test.go b/pkg/volume/csi/csi_attacher_test.go index ebf71093a44..bfbf0779057 100644 --- a/pkg/volume/csi/csi_attacher_test.go +++ b/pkg/volume/csi/csi_attacher_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "testing" "time" @@ -360,9 +361,12 @@ func TestAttacherWithCSIDriver(t *testing.T) { t.Log("plugin is not attachable") return } - + var wg sync.WaitGroup + wg.Add(1) go func(volSpec *volume.Spec, expectAttach bool) { attachID, err := csiAttacher.Attach(volSpec, types.NodeName("node")) + defer wg.Done() + if err != nil { t.Errorf("Attach() failed: %s", err) } @@ -378,6 +382,7 @@ func TestAttacherWithCSIDriver(t *testing.T) { } markVolumeAttached(t, csiAttacher.k8s, fakeWatcher, expectedAttachID, status) } + wg.Wait() }) } } diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index f16d9c59e42..51bb55f3bb5 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -804,6 +804,24 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsPersi customVolumeName := "" epServiceName := "" + kubeClient := p.plugin.host.GetKubeClient() + if kubeClient == nil { + return nil, 0, "", fmt.Errorf("failed to get kube client to update endpoint") + } + + if len(p.provisionerConfig.customEpNamePrefix) == 0 { + epServiceName = string(p.options.PVC.UID) + } else { + epServiceName = p.provisionerConfig.customEpNamePrefix + "-" + string(p.options.PVC.UID) + } + epNamespace := p.options.PVC.Namespace + endpoint, service, err := p.createOrGetEndpointService(epNamespace, epServiceName, p.options.PVC) + if err != nil { + klog.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) + return nil, 0, "", fmt.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) + } + klog.V(3).Infof("dynamic endpoint %v and service %v ", endpoint, service) + capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] // GlusterFS/heketi creates volumes in units of GiB. @@ -850,21 +868,42 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsPersi return nil, 0, "", fmt.Errorf("failed to get cluster nodes for volume %s: %v", volume, err) } - if len(p.provisionerConfig.customEpNamePrefix) == 0 { - epServiceName = string(p.options.PVC.UID) - } else { - epServiceName = p.provisionerConfig.customEpNamePrefix + "-" + string(p.options.PVC.UID) + addrlist := make([]v1.EndpointAddress, len(dynamicHostIps)) + for i, v := range dynamicHostIps { + addrlist[i].IP = v } - epNamespace := p.options.PVC.Namespace - endpoint, service, err := p.createEndpointService(epNamespace, epServiceName, dynamicHostIps, p.options.PVC) + subset := make([]v1.EndpointSubset, 1) + ports := []v1.EndpointPort{{Port: 1, Protocol: "TCP"}} + + endpoint.Subsets = subset + endpoint.Subsets[0].Addresses = addrlist + endpoint.Subsets[0].Ports = ports + + _, err = kubeClient.CoreV1().Endpoints(epNamespace).Update(endpoint) if err != nil { deleteErr := cli.VolumeDelete(volume.Id) if deleteErr != nil { klog.Errorf("failed to delete volume: %v, manual deletion of the volume required", deleteErr) } - return nil, 0, "", fmt.Errorf("failed to create endpoint/service %v/%v: %v", epNamespace, epServiceName, err) + + klog.V(3).Infof("failed to update endpoint, deleting %s", endpoint) + + err = kubeClient.CoreV1().Services(epNamespace).Delete(epServiceName, nil) + + if err != nil && errors.IsNotFound(err) { + klog.V(1).Infof("service %s does not exist in namespace %s", epServiceName, epNamespace) + err = nil + } + if err != nil { + klog.Errorf("failed to delete service %s/%s: %v", epNamespace, epServiceName, err) + } + klog.V(1).Infof("service/endpoint: %s/%s deleted successfully", epNamespace, epServiceName) + return nil, 0, "", fmt.Errorf("failed to update endpoint %s: %v", endpoint, err) + } - klog.V(3).Infof("dynamic endpoint %v and service %v ", endpoint, service) + + klog.V(3).Infof("endpoint %s updated successfully", endpoint) + return &v1.GlusterfsPersistentVolumeSource{ EndpointsName: endpoint.Name, EndpointsNamespace: &epNamespace, @@ -873,11 +912,11 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsPersi }, sz, volID, nil } -// createEndpointService() makes sure an endpoint and service -// exist for the given namespace, PVC name, endpoint name, and -// set of IPs. I.e. the endpoint or service is only created +// createOrGetEndpointService() makes sure an endpoint and service +// exist for the given namespace, PVC name, endpoint name +// I.e. the endpoint or service is only created // if it does not exist yet. -func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epServiceName string, hostips []string, pvc *v1.PersistentVolumeClaim) (endpoint *v1.Endpoints, service *v1.Service, err error) { +func (p *glusterfsVolumeProvisioner) createOrGetEndpointService(namespace string, epServiceName string, pvc *v1.PersistentVolumeClaim) (endpoint *v1.Endpoints, service *v1.Service, err error) { pvcNameOrID := "" if len(pvc.Name) >= 63 { pvcNameOrID = string(pvc.UID) @@ -885,10 +924,6 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS pvcNameOrID = pvc.Name } - addrlist := make([]v1.EndpointAddress, len(hostips)) - for i, v := range hostips { - addrlist[i].IP = v - } endpoint = &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -897,10 +932,6 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS "gluster.kubernetes.io/provisioned-for-pvc": pvcNameOrID, }, }, - Subsets: []v1.EndpointSubset{{ - Addresses: addrlist, - Ports: []v1.EndpointPort{{Port: 1, Protocol: "TCP"}}, - }}, } kubeClient := p.plugin.host.GetKubeClient() if kubeClient == nil { diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 049d10ce8c2..2c8e34e4d66 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -907,6 +907,11 @@ func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error if nodeName == UncertainAttachNode { return "/dev/vdb-test", nil } + // even if volume was previously attached to time out, we need to keep returning error + // so as reconciler can not confirm this volume as attached. + if nodeName == TimeoutAttachNode { + return "", fmt.Errorf("Timed out to attach volume %q to node %q", volumeName, nodeName) + } if volumeNode == nodeName || volumeNode == MultiAttachNode || nodeName == MultiAttachNode { return "/dev/vdb-test", nil } diff --git a/pkg/volume/util/nsenter/nsenter_mount.go b/pkg/volume/util/nsenter/nsenter_mount.go index 91145fa9d10..f756dc4287b 100644 --- a/pkg/volume/util/nsenter/nsenter_mount.go +++ b/pkg/volume/util/nsenter/nsenter_mount.go @@ -259,7 +259,7 @@ func (n *Mounter) GetFileType(pathname string) (mount.FileType, error) { return mount.FileTypeBlockDev, nil case "directory": return mount.FileTypeDirectory, nil - case "regular file": + case "regular file", "regular empty file": return mount.FileTypeFile, nil } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index 18a3d4aabd0..f82cd3d5903 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -138,6 +138,7 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) rbacv1helpers.NewRule("get", "list", "watch").Groups(policyGroup).Resources("poddisruptionbudgets").RuleOrDie(), rbacv1helpers.NewRule("get", "list", "watch").Groups(appsGroup).Resources("statefulsets").RuleOrDie(), rbacv1helpers.NewRule("update").Groups(policyGroup).Resources("poddisruptionbudgets/status").RuleOrDie(), + rbacv1helpers.NewRule("get").Groups("*").Resources("*/scale").RuleOrDie(), eventsRule(), }, }) @@ -230,7 +231,8 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) // glusterfs rbacv1helpers.NewRule("get", "list", "watch").Groups(storageGroup).Resources("storageclasses").RuleOrDie(), - rbacv1helpers.NewRule("get", "create", "delete").Groups(legacyGroup).Resources("services", "endpoints").RuleOrDie(), + rbacv1helpers.NewRule("get", "create", "update", "delete").Groups(legacyGroup).Resources("endpoints").RuleOrDie(), + rbacv1helpers.NewRule("get", "create", "delete").Groups(legacyGroup).Resources("services").RuleOrDie(), rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("secrets").RuleOrDie(), // openstack rbacv1helpers.NewRule("get", "list").Groups(legacyGroup).Resources("nodes").RuleOrDie(), diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy_test.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy_test.go index f1064f27084..365b5c3d20d 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy_test.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy_test.go @@ -33,6 +33,7 @@ var rolesWithAllowStar = sets.NewString( saRolePrefix+"resourcequota-controller", saRolePrefix+"horizontal-pod-autoscaler", saRolePrefix+"clusterrole-aggregation-controller", + saRolePrefix+"disruption-controller", ) // TestNoStarsForControllers confirms that no controller role has star verbs, groups, diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index 1384c1eea68..411a2749d11 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -382,6 +382,12 @@ items: - poddisruptionbudgets/status verbs: - update + - apiGroups: + - '*' + resources: + - '*/scale' + verbs: + - get - apiGroups: - "" resources: @@ -789,6 +795,14 @@ items: - "" resources: - endpoints + verbs: + - create + - delete + - get + - update + - apiGroups: + - "" + resources: - services verbs: - create diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index 218e438899c..d579b2fbeaa 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -11154,6 +11154,18 @@ func (m *WindowsSecurityContextOptions) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.GMSACredentialSpecName != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.GMSACredentialSpecName))) + i += copy(dAtA[i:], *m.GMSACredentialSpecName) + } + if m.GMSACredentialSpec != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.GMSACredentialSpec))) + i += copy(dAtA[i:], *m.GMSACredentialSpec) + } return i, nil } @@ -14764,6 +14776,14 @@ func (m *WeightedPodAffinityTerm) Size() (n int) { func (m *WindowsSecurityContextOptions) Size() (n int) { var l int _ = l + if m.GMSACredentialSpecName != nil { + l = len(*m.GMSACredentialSpecName) + n += 1 + l + sovGenerated(uint64(l)) + } + if m.GMSACredentialSpec != nil { + l = len(*m.GMSACredentialSpec) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -17600,6 +17620,8 @@ func (this *WindowsSecurityContextOptions) String() string { return "nil" } s := strings.Join([]string{`&WindowsSecurityContextOptions{`, + `GMSACredentialSpecName:` + valueToStringGenerated(this.GMSACredentialSpecName) + `,`, + `GMSACredentialSpec:` + valueToStringGenerated(this.GMSACredentialSpec) + `,`, `}`, }, "") return s @@ -52282,6 +52304,66 @@ func (m *WindowsSecurityContextOptions) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: WindowsSecurityContextOptions: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GMSACredentialSpecName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.GMSACredentialSpecName = &s + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GMSACredentialSpec", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.GMSACredentialSpec = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -52413,819 +52495,821 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 13009 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x90, 0x24, 0x47, - 0x56, 0xd8, 0x55, 0xf7, 0x7c, 0x74, 0xbf, 0xf9, 0xce, 0xdd, 0x95, 0x66, 0x47, 0xbb, 0xdb, 0xab, - 0xd2, 0xdd, 0x6a, 0x75, 0x92, 0x66, 0x4f, 0x2b, 0xe9, 0x4e, 0x9c, 0x74, 0x82, 0x99, 0xe9, 0x99, - 0xdd, 0xd6, 0xee, 0xcc, 0xb6, 0xb2, 0x67, 0x77, 0xef, 0x84, 0xee, 0xb8, 0x9a, 0xae, 0x9c, 0x99, - 0xd2, 0xf4, 0x54, 0xb5, 0xaa, 0xaa, 0x67, 0x77, 0x64, 0x08, 0xe3, 0xc3, 0x60, 0x2e, 0xc0, 0x8e, - 0x0b, 0xfb, 0xc2, 0x1f, 0x40, 0xe0, 0x08, 0x8c, 0x03, 0x30, 0xd8, 0x61, 0x0c, 0x06, 0xcc, 0x61, - 0x1b, 0x83, 0x1d, 0x81, 0xfd, 0xe3, 0x8c, 0x1d, 0xe1, 0x38, 0x22, 0x08, 0x8f, 0x61, 0x71, 0x98, - 0xe0, 0x87, 0xc1, 0x61, 0xfc, 0x87, 0x31, 0x61, 0x1c, 0xf9, 0x59, 0x99, 0xd5, 0x55, 0xdd, 0x3d, - 0xab, 0xd9, 0x91, 0x20, 0xee, 0x5f, 0x77, 0xbe, 0x97, 0x2f, 0xb3, 0xf2, 0xf3, 0xbd, 0x97, 0xef, - 0x03, 0x5e, 0xdd, 0x79, 0x25, 0x9a, 0xf7, 0x82, 0x2b, 0x3b, 0x9d, 0x0d, 0x12, 0xfa, 0x24, 0x26, - 0xd1, 0x95, 0x3d, 0xe2, 0xbb, 0x41, 0x78, 0x45, 0x00, 0x9c, 0xb6, 0x77, 0xa5, 0x19, 0x84, 0xe4, - 0xca, 0xde, 0x0b, 0x57, 0xb6, 0x88, 0x4f, 0x42, 0x27, 0x26, 0xee, 0x7c, 0x3b, 0x0c, 0xe2, 0x00, - 0x21, 0x8e, 0x33, 0xef, 0xb4, 0xbd, 0x79, 0x8a, 0x33, 0xbf, 0xf7, 0xc2, 0xdc, 0xf3, 0x5b, 0x5e, - 0xbc, 0xdd, 0xd9, 0x98, 0x6f, 0x06, 0xbb, 0x57, 0xb6, 0x82, 0xad, 0xe0, 0x0a, 0x43, 0xdd, 0xe8, - 0x6c, 0xb2, 0x7f, 0xec, 0x0f, 0xfb, 0xc5, 0x49, 0xcc, 0xbd, 0x94, 0x34, 0xb3, 0xeb, 0x34, 0xb7, - 0x3d, 0x9f, 0x84, 0xfb, 0x57, 0xda, 0x3b, 0x5b, 0xac, 0xdd, 0x90, 0x44, 0x41, 0x27, 0x6c, 0x92, - 0x74, 0xc3, 0x3d, 0x6b, 0x45, 0x57, 0x76, 0x49, 0xec, 0x64, 0x74, 0x77, 0xee, 0x4a, 0x5e, 0xad, - 0xb0, 0xe3, 0xc7, 0xde, 0x6e, 0x77, 0x33, 0x9f, 0xec, 0x57, 0x21, 0x6a, 0x6e, 0x93, 0x5d, 0xa7, - 0xab, 0xde, 0x8b, 0x79, 0xf5, 0x3a, 0xb1, 0xd7, 0xba, 0xe2, 0xf9, 0x71, 0x14, 0x87, 0xe9, 0x4a, - 0xf6, 0x37, 0x2c, 0xb8, 0xb8, 0x70, 0xb7, 0xb1, 0xdc, 0x72, 0xa2, 0xd8, 0x6b, 0x2e, 0xb6, 0x82, - 0xe6, 0x4e, 0x23, 0x0e, 0x42, 0x72, 0x27, 0x68, 0x75, 0x76, 0x49, 0x83, 0x0d, 0x04, 0x7a, 0x0e, - 0x4a, 0x7b, 0xec, 0x7f, 0xad, 0x3a, 0x6b, 0x5d, 0xb4, 0x2e, 0x97, 0x17, 0xa7, 0x7f, 0xe3, 0xa0, - 0xf2, 0x91, 0x07, 0x07, 0x95, 0xd2, 0x1d, 0x51, 0x8e, 0x15, 0x06, 0xba, 0x04, 0x23, 0x9b, 0xd1, - 0xfa, 0x7e, 0x9b, 0xcc, 0x16, 0x18, 0xee, 0xa4, 0xc0, 0x1d, 0x59, 0x69, 0xd0, 0x52, 0x2c, 0xa0, - 0xe8, 0x0a, 0x94, 0xdb, 0x4e, 0x18, 0x7b, 0xb1, 0x17, 0xf8, 0xb3, 0xc5, 0x8b, 0xd6, 0xe5, 0xe1, - 0xc5, 0x19, 0x81, 0x5a, 0xae, 0x4b, 0x00, 0x4e, 0x70, 0x68, 0x37, 0x42, 0xe2, 0xb8, 0xb7, 0xfc, - 0xd6, 0xfe, 0xec, 0xd0, 0x45, 0xeb, 0x72, 0x29, 0xe9, 0x06, 0x16, 0xe5, 0x58, 0x61, 0xd8, 0x3f, - 0x54, 0x80, 0xd2, 0xc2, 0xe6, 0xa6, 0xe7, 0x7b, 0xf1, 0x3e, 0xba, 0x03, 0xe3, 0x7e, 0xe0, 0x12, - 0xf9, 0x9f, 0x7d, 0xc5, 0xd8, 0xd5, 0x8b, 0xf3, 0xdd, 0x4b, 0x69, 0x7e, 0x4d, 0xc3, 0x5b, 0x9c, - 0x7e, 0x70, 0x50, 0x19, 0xd7, 0x4b, 0xb0, 0x41, 0x07, 0x61, 0x18, 0x6b, 0x07, 0xae, 0x22, 0x5b, - 0x60, 0x64, 0x2b, 0x59, 0x64, 0xeb, 0x09, 0xda, 0xe2, 0xd4, 0x83, 0x83, 0xca, 0x98, 0x56, 0x80, - 0x75, 0x22, 0x68, 0x03, 0xa6, 0xe8, 0x5f, 0x3f, 0xf6, 0x14, 0xdd, 0x22, 0xa3, 0xfb, 0x54, 0x1e, - 0x5d, 0x0d, 0x75, 0xf1, 0xd4, 0x83, 0x83, 0xca, 0x54, 0xaa, 0x10, 0xa7, 0x09, 0xda, 0xef, 0xc1, - 0xe4, 0x42, 0x1c, 0x3b, 0xcd, 0x6d, 0xe2, 0xf2, 0x19, 0x44, 0x2f, 0xc1, 0x90, 0xef, 0xec, 0x12, - 0x31, 0xbf, 0x17, 0xc5, 0xc0, 0x0e, 0xad, 0x39, 0xbb, 0xe4, 0xf0, 0xa0, 0x32, 0x7d, 0xdb, 0xf7, - 0xde, 0xed, 0x88, 0x55, 0x41, 0xcb, 0x30, 0xc3, 0x46, 0x57, 0x01, 0x5c, 0xb2, 0xe7, 0x35, 0x49, - 0xdd, 0x89, 0xb7, 0xc5, 0x7c, 0x23, 0x51, 0x17, 0xaa, 0x0a, 0x82, 0x35, 0x2c, 0xfb, 0x3e, 0x94, - 0x17, 0xf6, 0x02, 0xcf, 0xad, 0x07, 0x6e, 0x84, 0x76, 0x60, 0xaa, 0x1d, 0x92, 0x4d, 0x12, 0xaa, - 0xa2, 0x59, 0xeb, 0x62, 0xf1, 0xf2, 0xd8, 0xd5, 0xcb, 0x99, 0x1f, 0x6b, 0xa2, 0x2e, 0xfb, 0x71, - 0xb8, 0xbf, 0xf8, 0xb8, 0x68, 0x6f, 0x2a, 0x05, 0xc5, 0x69, 0xca, 0xf6, 0xbf, 0x2d, 0xc0, 0x99, - 0x85, 0xf7, 0x3a, 0x21, 0xa9, 0x7a, 0xd1, 0x4e, 0x7a, 0x85, 0xbb, 0x5e, 0xb4, 0xb3, 0x96, 0x8c, - 0x80, 0x5a, 0x5a, 0x55, 0x51, 0x8e, 0x15, 0x06, 0x7a, 0x1e, 0x46, 0xe9, 0xef, 0xdb, 0xb8, 0x26, - 0x3e, 0xf9, 0x94, 0x40, 0x1e, 0xab, 0x3a, 0xb1, 0x53, 0xe5, 0x20, 0x2c, 0x71, 0xd0, 0x2a, 0x8c, - 0x35, 0xd9, 0x86, 0xdc, 0x5a, 0x0d, 0x5c, 0xc2, 0x26, 0xb3, 0xbc, 0xf8, 0x2c, 0x45, 0x5f, 0x4a, - 0x8a, 0x0f, 0x0f, 0x2a, 0xb3, 0xbc, 0x6f, 0x82, 0x84, 0x06, 0xc3, 0x7a, 0x7d, 0x64, 0xab, 0xfd, - 0x35, 0xc4, 0x28, 0x41, 0xc6, 0xde, 0xba, 0xac, 0x6d, 0x95, 0x61, 0xb6, 0x55, 0xc6, 0xb3, 0xb7, - 0x09, 0x7a, 0x01, 0x86, 0x76, 0x3c, 0xdf, 0x9d, 0x1d, 0x61, 0xb4, 0xce, 0xd3, 0x39, 0xbf, 0xe1, - 0xf9, 0xee, 0xe1, 0x41, 0x65, 0xc6, 0xe8, 0x0e, 0x2d, 0xc4, 0x0c, 0xd5, 0xfe, 0x63, 0x0b, 0x2a, - 0x0c, 0xb6, 0xe2, 0xb5, 0x48, 0x9d, 0x84, 0x91, 0x17, 0xc5, 0xc4, 0x8f, 0x8d, 0x01, 0xbd, 0x0a, - 0x10, 0x91, 0x66, 0x48, 0x62, 0x6d, 0x48, 0xd5, 0xc2, 0x68, 0x28, 0x08, 0xd6, 0xb0, 0xe8, 0x81, - 0x10, 0x6d, 0x3b, 0x21, 0x5b, 0x5f, 0x62, 0x60, 0xd5, 0x81, 0xd0, 0x90, 0x00, 0x9c, 0xe0, 0x18, - 0x07, 0x42, 0xb1, 0xdf, 0x81, 0x80, 0x3e, 0x03, 0x53, 0x49, 0x63, 0x51, 0xdb, 0x69, 0xca, 0x01, - 0x64, 0x5b, 0xa6, 0x61, 0x82, 0x70, 0x1a, 0xd7, 0xfe, 0x47, 0x96, 0x58, 0x3c, 0xf4, 0xab, 0x3f, - 0xe4, 0xdf, 0x6a, 0xff, 0x92, 0x05, 0xa3, 0x8b, 0x9e, 0xef, 0x7a, 0xfe, 0x16, 0xfa, 0x22, 0x94, - 0xe8, 0xdd, 0xe4, 0x3a, 0xb1, 0x23, 0xce, 0xbd, 0x4f, 0x68, 0x7b, 0x4b, 0x5d, 0x15, 0xf3, 0xed, - 0x9d, 0x2d, 0x5a, 0x10, 0xcd, 0x53, 0x6c, 0xba, 0xdb, 0x6e, 0x6d, 0xbc, 0x43, 0x9a, 0xf1, 0x2a, - 0x89, 0x9d, 0xe4, 0x73, 0x92, 0x32, 0xac, 0xa8, 0xa2, 0x1b, 0x30, 0x12, 0x3b, 0xe1, 0x16, 0x89, - 0xc5, 0x01, 0x98, 0x79, 0x50, 0xf1, 0x9a, 0x98, 0xee, 0x48, 0xe2, 0x37, 0x49, 0x72, 0x2d, 0xac, - 0xb3, 0xaa, 0x58, 0x90, 0xb0, 0xff, 0xfa, 0x28, 0x9c, 0x5d, 0x6a, 0xd4, 0x72, 0xd6, 0xd5, 0x25, - 0x18, 0x71, 0x43, 0x6f, 0x8f, 0x84, 0x62, 0x9c, 0x15, 0x95, 0x2a, 0x2b, 0xc5, 0x02, 0x8a, 0x5e, - 0x81, 0x71, 0x7e, 0x21, 0x5d, 0x77, 0x7c, 0xb7, 0x25, 0x87, 0xf8, 0xb4, 0xc0, 0x1e, 0xbf, 0xa3, - 0xc1, 0xb0, 0x81, 0x79, 0xc4, 0x45, 0x75, 0x29, 0xb5, 0x19, 0xf3, 0x2e, 0xbb, 0x2f, 0x5b, 0x30, - 0xcd, 0x9b, 0x59, 0x88, 0xe3, 0xd0, 0xdb, 0xe8, 0xc4, 0x24, 0x9a, 0x1d, 0x66, 0x27, 0xdd, 0x52, - 0xd6, 0x68, 0xe5, 0x8e, 0xc0, 0xfc, 0x9d, 0x14, 0x15, 0x7e, 0x08, 0xce, 0x8a, 0x76, 0xa7, 0xd3, - 0x60, 0xdc, 0xd5, 0x2c, 0xfa, 0x1e, 0x0b, 0xe6, 0x9a, 0x81, 0x1f, 0x87, 0x41, 0xab, 0x45, 0xc2, - 0x7a, 0x67, 0xa3, 0xe5, 0x45, 0xdb, 0x7c, 0x9d, 0x62, 0xb2, 0xc9, 0x4e, 0x82, 0x9c, 0x39, 0x54, - 0x48, 0x62, 0x0e, 0x2f, 0x3c, 0x38, 0xa8, 0xcc, 0x2d, 0xe5, 0x92, 0xc2, 0x3d, 0x9a, 0x41, 0x3b, - 0x80, 0xe8, 0x55, 0xda, 0x88, 0x9d, 0x2d, 0x92, 0x34, 0x3e, 0x3a, 0x78, 0xe3, 0x8f, 0x3d, 0x38, - 0xa8, 0xa0, 0xb5, 0x2e, 0x12, 0x38, 0x83, 0x2c, 0x7a, 0x17, 0x4e, 0xd3, 0xd2, 0xae, 0x6f, 0x2d, - 0x0d, 0xde, 0xdc, 0xec, 0x83, 0x83, 0xca, 0xe9, 0xb5, 0x0c, 0x22, 0x38, 0x93, 0x34, 0xfa, 0x6e, - 0x0b, 0xce, 0x26, 0x9f, 0xbf, 0x7c, 0xbf, 0xed, 0xf8, 0x6e, 0xd2, 0x70, 0x79, 0xf0, 0x86, 0xe9, - 0x99, 0x7c, 0x76, 0x29, 0x8f, 0x12, 0xce, 0x6f, 0x64, 0x6e, 0x09, 0xce, 0x64, 0xae, 0x16, 0x34, - 0x0d, 0xc5, 0x1d, 0xc2, 0xb9, 0xa0, 0x32, 0xa6, 0x3f, 0xd1, 0x69, 0x18, 0xde, 0x73, 0x5a, 0x1d, - 0xb1, 0x51, 0x30, 0xff, 0xf3, 0xe9, 0xc2, 0x2b, 0x96, 0xfd, 0xef, 0x8a, 0x30, 0xb5, 0xd4, 0xa8, - 0x3d, 0xd4, 0x2e, 0xd4, 0xaf, 0xa1, 0x42, 0xcf, 0x6b, 0x28, 0xb9, 0xd4, 0x8a, 0xb9, 0x97, 0xda, - 0x5f, 0xce, 0xd8, 0x42, 0x43, 0x6c, 0x0b, 0x7d, 0x4b, 0xce, 0x16, 0x3a, 0xe6, 0x8d, 0xb3, 0x97, - 0xb3, 0x8a, 0x86, 0xd9, 0x64, 0x66, 0x72, 0x2c, 0x37, 0x83, 0xa6, 0xd3, 0x4a, 0x1f, 0x7d, 0x47, - 0x5c, 0x4a, 0xc7, 0x33, 0x8f, 0x4d, 0x18, 0x5f, 0x72, 0xda, 0xce, 0x86, 0xd7, 0xf2, 0x62, 0x8f, - 0x44, 0xe8, 0x69, 0x28, 0x3a, 0xae, 0xcb, 0xb8, 0xad, 0xf2, 0xe2, 0x99, 0x07, 0x07, 0x95, 0xe2, - 0x82, 0x4b, 0xaf, 0x7d, 0x50, 0x58, 0xfb, 0x98, 0x62, 0xa0, 0x8f, 0xc3, 0x90, 0x1b, 0x06, 0xed, - 0xd9, 0x02, 0xc3, 0xa4, 0xbb, 0x6e, 0xa8, 0x1a, 0x06, 0xed, 0x14, 0x2a, 0xc3, 0xb1, 0x7f, 0xb5, - 0x00, 0xe7, 0x96, 0x48, 0x7b, 0x7b, 0xa5, 0x91, 0x73, 0x7e, 0x5f, 0x86, 0xd2, 0x6e, 0xe0, 0x7b, - 0x71, 0x10, 0x46, 0xa2, 0x69, 0xb6, 0x22, 0x56, 0x45, 0x19, 0x56, 0x50, 0x74, 0x11, 0x86, 0xda, - 0x09, 0x53, 0x39, 0x2e, 0x19, 0x52, 0xc6, 0x4e, 0x32, 0x08, 0xc5, 0xe8, 0x44, 0x24, 0x14, 0x2b, - 0x46, 0x61, 0xdc, 0x8e, 0x48, 0x88, 0x19, 0x24, 0xb9, 0x99, 0xe9, 0x9d, 0x2d, 0x4e, 0xe8, 0xd4, - 0xcd, 0x4c, 0x21, 0x58, 0xc3, 0x42, 0x75, 0x28, 0x47, 0xa9, 0x99, 0x1d, 0x68, 0x9b, 0x4e, 0xb0, - 0xab, 0x5b, 0xcd, 0x64, 0x42, 0xc4, 0xb8, 0x51, 0x46, 0xfa, 0x5e, 0xdd, 0x5f, 0x2b, 0x00, 0xe2, - 0x43, 0xf8, 0xe7, 0x6c, 0xe0, 0x6e, 0x77, 0x0f, 0xdc, 0xe0, 0x5b, 0xe2, 0xb8, 0x46, 0xef, 0xff, - 0x58, 0x70, 0x6e, 0xc9, 0xf3, 0x5d, 0x12, 0xe6, 0x2c, 0xc0, 0x47, 0x23, 0xcb, 0x1e, 0x8d, 0x69, - 0x30, 0x96, 0xd8, 0xd0, 0x31, 0x2c, 0x31, 0xfb, 0x8f, 0x2c, 0x40, 0xfc, 0xb3, 0x3f, 0x74, 0x1f, - 0x7b, 0xbb, 0xfb, 0x63, 0x8f, 0x61, 0x59, 0xd8, 0x37, 0x61, 0x72, 0xa9, 0xe5, 0x11, 0x3f, 0xae, - 0xd5, 0x97, 0x02, 0x7f, 0xd3, 0xdb, 0x42, 0x9f, 0x86, 0xc9, 0xd8, 0xdb, 0x25, 0x41, 0x27, 0x6e, - 0x90, 0x66, 0xe0, 0x33, 0x49, 0xd2, 0xba, 0x3c, 0xbc, 0x88, 0x1e, 0x1c, 0x54, 0x26, 0xd7, 0x0d, - 0x08, 0x4e, 0x61, 0xda, 0xbf, 0x4d, 0xc7, 0x2f, 0xd8, 0x6d, 0x07, 0x3e, 0xf1, 0xe3, 0xa5, 0xc0, - 0x77, 0xb9, 0xc6, 0xe1, 0xd3, 0x30, 0x14, 0xd3, 0xf1, 0xe0, 0x63, 0x77, 0x49, 0x6e, 0x14, 0x3a, - 0x0a, 0x87, 0x07, 0x95, 0xc7, 0xba, 0x6b, 0xb0, 0x71, 0x62, 0x75, 0xd0, 0xb7, 0xc0, 0x48, 0x14, - 0x3b, 0x71, 0x27, 0x12, 0xa3, 0xf9, 0xa4, 0x1c, 0xcd, 0x06, 0x2b, 0x3d, 0x3c, 0xa8, 0x4c, 0xa9, - 0x6a, 0xbc, 0x08, 0x8b, 0x0a, 0xe8, 0x19, 0x18, 0xdd, 0x25, 0x51, 0xe4, 0x6c, 0xc9, 0xdb, 0x70, - 0x4a, 0xd4, 0x1d, 0x5d, 0xe5, 0xc5, 0x58, 0xc2, 0xd1, 0x53, 0x30, 0x4c, 0xc2, 0x30, 0x08, 0xc5, - 0x1e, 0x9d, 0x10, 0x88, 0xc3, 0xcb, 0xb4, 0x10, 0x73, 0x98, 0xfd, 0x1f, 0x2d, 0x98, 0x52, 0x7d, - 0xe5, 0x6d, 0x9d, 0x80, 0x54, 0xf0, 0x16, 0x40, 0x53, 0x7e, 0x60, 0xc4, 0x6e, 0x8f, 0xb1, 0xab, - 0x97, 0x32, 0x2f, 0xea, 0xae, 0x61, 0x4c, 0x28, 0xab, 0xa2, 0x08, 0x6b, 0xd4, 0xec, 0x7f, 0x69, - 0xc1, 0xa9, 0xd4, 0x17, 0xdd, 0xf4, 0xa2, 0x18, 0xbd, 0xdd, 0xf5, 0x55, 0xf3, 0x83, 0x7d, 0x15, - 0xad, 0xcd, 0xbe, 0x49, 0x2d, 0x65, 0x59, 0xa2, 0x7d, 0xd1, 0x75, 0x18, 0xf6, 0x62, 0xb2, 0x2b, - 0x3f, 0xe6, 0xa9, 0x9e, 0x1f, 0xc3, 0x7b, 0x95, 0xcc, 0x48, 0x8d, 0xd6, 0xc4, 0x9c, 0x80, 0xfd, - 0xb7, 0x8a, 0x50, 0xe6, 0xcb, 0x76, 0xd5, 0x69, 0x9f, 0xc0, 0x5c, 0xd4, 0x60, 0x88, 0x51, 0xe7, - 0x1d, 0x7f, 0x3a, 0xbb, 0xe3, 0xa2, 0x3b, 0xf3, 0x54, 0xe4, 0xe7, 0xcc, 0x91, 0xba, 0x1a, 0x68, - 0x11, 0x66, 0x24, 0x90, 0x03, 0xb0, 0xe1, 0xf9, 0x4e, 0xb8, 0x4f, 0xcb, 0x66, 0x8b, 0x8c, 0xe0, - 0xf3, 0xbd, 0x09, 0x2e, 0x2a, 0x7c, 0x4e, 0x56, 0xf5, 0x35, 0x01, 0x60, 0x8d, 0xe8, 0xdc, 0xa7, - 0xa0, 0xac, 0x90, 0x8f, 0xc2, 0xe3, 0xcc, 0x7d, 0x06, 0xa6, 0x52, 0x6d, 0xf5, 0xab, 0x3e, 0xae, - 0xb3, 0x48, 0xbf, 0xcc, 0x4e, 0x01, 0xd1, 0xeb, 0x65, 0x7f, 0x4f, 0x9c, 0xa2, 0xef, 0xc1, 0xe9, - 0x56, 0xc6, 0xe1, 0x24, 0xa6, 0x6a, 0xf0, 0xc3, 0xec, 0x9c, 0xf8, 0xec, 0xd3, 0x59, 0x50, 0x9c, - 0xd9, 0x06, 0xbd, 0xf6, 0x83, 0x36, 0x5d, 0xf3, 0x4e, 0x4b, 0xe7, 0xa0, 0x6f, 0x89, 0x32, 0xac, - 0xa0, 0xf4, 0x08, 0x3b, 0xad, 0x3a, 0x7f, 0x83, 0xec, 0x37, 0x48, 0x8b, 0x34, 0xe3, 0x20, 0xfc, - 0x40, 0xbb, 0x7f, 0x9e, 0x8f, 0x3e, 0x3f, 0x01, 0xc7, 0x04, 0x81, 0xe2, 0x0d, 0xb2, 0xcf, 0xa7, - 0x42, 0xff, 0xba, 0x62, 0xcf, 0xaf, 0xfb, 0x59, 0x0b, 0x26, 0xd4, 0xd7, 0x9d, 0xc0, 0x56, 0x5f, - 0x34, 0xb7, 0xfa, 0xf9, 0x9e, 0x0b, 0x3c, 0x67, 0x93, 0x7f, 0xad, 0x00, 0x67, 0x15, 0x0e, 0x65, - 0xf7, 0xf9, 0x1f, 0xb1, 0xaa, 0xae, 0x40, 0xd9, 0x57, 0x8a, 0x28, 0xcb, 0xd4, 0x00, 0x25, 0x6a, - 0xa8, 0x04, 0x87, 0x72, 0x6d, 0x7e, 0xa2, 0x2d, 0x1a, 0xd7, 0x35, 0xb4, 0x42, 0x1b, 0xbb, 0x08, - 0xc5, 0x8e, 0xe7, 0x8a, 0x3b, 0xe3, 0x13, 0x72, 0xb4, 0x6f, 0xd7, 0xaa, 0x87, 0x07, 0x95, 0x27, - 0xf3, 0x5e, 0x07, 0xe8, 0x65, 0x15, 0xcd, 0xdf, 0xae, 0x55, 0x31, 0xad, 0x8c, 0x16, 0x60, 0x4a, - 0x3e, 0x80, 0xdc, 0xa1, 0x1c, 0x54, 0xe0, 0x8b, 0xab, 0x45, 0xa9, 0x59, 0xb1, 0x09, 0xc6, 0x69, - 0x7c, 0x54, 0x85, 0xe9, 0x9d, 0xce, 0x06, 0x69, 0x91, 0x98, 0x7f, 0xf0, 0x0d, 0xc2, 0x95, 0x90, - 0xe5, 0x44, 0xd8, 0xba, 0x91, 0x82, 0xe3, 0xae, 0x1a, 0xf6, 0x9f, 0xb1, 0x23, 0x5e, 0x8c, 0x5e, - 0x3d, 0x0c, 0xe8, 0xc2, 0xa2, 0xd4, 0x3f, 0xc8, 0xe5, 0x3c, 0xc8, 0xaa, 0xb8, 0x41, 0xf6, 0xd7, - 0x03, 0xca, 0x6c, 0x67, 0xaf, 0x0a, 0x63, 0xcd, 0x0f, 0xf5, 0x5c, 0xf3, 0x3f, 0x5f, 0x80, 0x33, - 0x6a, 0x04, 0x0c, 0xbe, 0xee, 0xcf, 0xfb, 0x18, 0xbc, 0x00, 0x63, 0x2e, 0xd9, 0x74, 0x3a, 0xad, - 0x58, 0x69, 0xc4, 0x87, 0xf9, 0xab, 0x48, 0x35, 0x29, 0xc6, 0x3a, 0xce, 0x11, 0x86, 0xed, 0xc7, - 0xc7, 0xd8, 0xdd, 0x1a, 0x3b, 0x74, 0x8d, 0xab, 0x5d, 0x63, 0xe5, 0xee, 0x9a, 0xa7, 0x60, 0xd8, - 0xdb, 0xa5, 0xbc, 0x56, 0xc1, 0x64, 0xa1, 0x6a, 0xb4, 0x10, 0x73, 0x18, 0xfa, 0x18, 0x8c, 0x36, - 0x83, 0xdd, 0x5d, 0xc7, 0x77, 0xd9, 0x95, 0x57, 0x5e, 0x1c, 0xa3, 0xec, 0xd8, 0x12, 0x2f, 0xc2, - 0x12, 0x86, 0xce, 0xc1, 0x90, 0x13, 0x6e, 0x71, 0xb5, 0x44, 0x79, 0xb1, 0x44, 0x5b, 0x5a, 0x08, - 0xb7, 0x22, 0xcc, 0x4a, 0xa9, 0x54, 0x75, 0x2f, 0x08, 0x77, 0x3c, 0x7f, 0xab, 0xea, 0x85, 0x62, - 0x4b, 0xa8, 0xbb, 0xf0, 0xae, 0x82, 0x60, 0x0d, 0x0b, 0xad, 0xc0, 0x70, 0x3b, 0x08, 0xe3, 0x68, - 0x76, 0x84, 0x0d, 0xf7, 0x93, 0x39, 0x07, 0x11, 0xff, 0xda, 0x7a, 0x10, 0xc6, 0xc9, 0x07, 0xd0, - 0x7f, 0x11, 0xe6, 0xd5, 0xd1, 0xb7, 0x40, 0x91, 0xf8, 0x7b, 0xb3, 0xa3, 0x8c, 0xca, 0x5c, 0x16, - 0x95, 0x65, 0x7f, 0xef, 0x8e, 0x13, 0x26, 0xa7, 0xf4, 0xb2, 0xbf, 0x87, 0x69, 0x1d, 0xf4, 0x39, - 0x28, 0xcb, 0x2d, 0x1e, 0x09, 0x8d, 0x59, 0xe6, 0x12, 0x93, 0x07, 0x03, 0x26, 0xef, 0x76, 0xbc, - 0x90, 0xec, 0x12, 0x3f, 0x8e, 0x92, 0x33, 0x4d, 0x42, 0x23, 0x9c, 0x50, 0x43, 0x9f, 0x93, 0x6a, - 0xda, 0xd5, 0xa0, 0xe3, 0xc7, 0xd1, 0x6c, 0x99, 0x75, 0x2f, 0xf3, 0x01, 0xed, 0x4e, 0x82, 0x97, - 0xd6, 0xe3, 0xf2, 0xca, 0xd8, 0x20, 0x85, 0x30, 0x4c, 0xb4, 0xbc, 0x3d, 0xe2, 0x93, 0x28, 0xaa, - 0x87, 0xc1, 0x06, 0x99, 0x05, 0xd6, 0xf3, 0xb3, 0xd9, 0xef, 0x4a, 0xc1, 0x06, 0x59, 0x9c, 0x79, - 0x70, 0x50, 0x99, 0xb8, 0xa9, 0xd7, 0xc1, 0x26, 0x09, 0x74, 0x1b, 0x26, 0xa9, 0x5c, 0xe3, 0x25, - 0x44, 0xc7, 0xfa, 0x11, 0x65, 0xd2, 0x07, 0x36, 0x2a, 0xe1, 0x14, 0x11, 0xf4, 0x06, 0x94, 0x5b, - 0xde, 0x26, 0x69, 0xee, 0x37, 0x5b, 0x64, 0x76, 0x9c, 0x51, 0xcc, 0xdc, 0x56, 0x37, 0x25, 0x12, - 0x97, 0x8b, 0xd4, 0x5f, 0x9c, 0x54, 0x47, 0x77, 0xe0, 0xb1, 0x98, 0x84, 0xbb, 0x9e, 0xef, 0xd0, - 0xed, 0x20, 0xe4, 0x05, 0xf6, 0x3a, 0x37, 0xc1, 0xd6, 0xdb, 0x05, 0x31, 0x74, 0x8f, 0xad, 0x67, - 0x62, 0xe1, 0x9c, 0xda, 0xe8, 0x16, 0x4c, 0xb1, 0x9d, 0x50, 0xef, 0xb4, 0x5a, 0xf5, 0xa0, 0xe5, - 0x35, 0xf7, 0x67, 0x27, 0x19, 0xc1, 0x8f, 0xc9, 0x7b, 0xa1, 0x66, 0x82, 0x0f, 0x0f, 0x2a, 0x90, - 0xfc, 0xc3, 0xe9, 0xda, 0x68, 0x83, 0x3d, 0xc7, 0x74, 0x42, 0x2f, 0xde, 0xa7, 0xeb, 0x97, 0xdc, - 0x8f, 0x67, 0xa7, 0x7a, 0x8a, 0xc2, 0x3a, 0xaa, 0x7a, 0xb3, 0xd1, 0x0b, 0x71, 0x9a, 0x20, 0xdd, - 0xda, 0x51, 0xec, 0x7a, 0xfe, 0xec, 0x34, 0x3b, 0x31, 0xd4, 0xce, 0x68, 0xd0, 0x42, 0xcc, 0x61, - 0xec, 0x29, 0x86, 0xfe, 0xb8, 0x45, 0x4f, 0xd0, 0x19, 0x86, 0x98, 0x3c, 0xc5, 0x48, 0x00, 0x4e, - 0x70, 0x28, 0x53, 0x13, 0xc7, 0xfb, 0xb3, 0x88, 0xa1, 0xaa, 0xed, 0xb2, 0xbe, 0xfe, 0x39, 0x4c, - 0xcb, 0xd1, 0x4d, 0x18, 0x25, 0xfe, 0xde, 0x4a, 0x18, 0xec, 0xce, 0x9e, 0xca, 0xdf, 0xb3, 0xcb, - 0x1c, 0x85, 0x1f, 0xe8, 0x89, 0x80, 0x27, 0x8a, 0xb1, 0x24, 0x81, 0xee, 0xc3, 0x6c, 0xc6, 0x8c, - 0xf0, 0x09, 0x38, 0xcd, 0x26, 0xe0, 0x35, 0x51, 0x77, 0x76, 0x3d, 0x07, 0xef, 0xb0, 0x07, 0x0c, - 0xe7, 0x52, 0x47, 0x9f, 0x87, 0x09, 0xbe, 0xa1, 0xf8, 0x3b, 0x6e, 0x34, 0x7b, 0x86, 0x7d, 0xcd, - 0xc5, 0xfc, 0xcd, 0xc9, 0x11, 0x17, 0xcf, 0x88, 0x0e, 0x4d, 0xe8, 0xa5, 0x11, 0x36, 0xa9, 0xd9, - 0x1b, 0x30, 0xa9, 0xce, 0x2d, 0xb6, 0x74, 0x50, 0x05, 0x86, 0x19, 0xb7, 0x23, 0xf4, 0x5b, 0x65, - 0x3a, 0x53, 0x8c, 0x13, 0xc2, 0xbc, 0x9c, 0xcd, 0x94, 0xf7, 0x1e, 0x59, 0xdc, 0x8f, 0x09, 0x97, - 0xaa, 0x8b, 0xda, 0x4c, 0x49, 0x00, 0x4e, 0x70, 0xec, 0xff, 0xc7, 0xb9, 0xc6, 0xe4, 0x70, 0x1c, - 0xe0, 0x3a, 0x78, 0x0e, 0x4a, 0xdb, 0x41, 0x14, 0x53, 0x6c, 0xd6, 0xc6, 0x70, 0xc2, 0x27, 0x5e, - 0x17, 0xe5, 0x58, 0x61, 0xa0, 0x57, 0x61, 0xa2, 0xa9, 0x37, 0x20, 0xee, 0x32, 0x35, 0x04, 0x46, - 0xeb, 0xd8, 0xc4, 0x45, 0xaf, 0x40, 0x89, 0x59, 0x61, 0x34, 0x83, 0x96, 0x60, 0xb2, 0xe4, 0x85, - 0x5c, 0xaa, 0x8b, 0xf2, 0x43, 0xed, 0x37, 0x56, 0xd8, 0xe8, 0x12, 0x8c, 0xd0, 0x2e, 0xd4, 0xea, - 0xe2, 0x16, 0x51, 0xaa, 0x9a, 0xeb, 0xac, 0x14, 0x0b, 0xa8, 0xfd, 0x37, 0x0b, 0xda, 0x28, 0x53, - 0x89, 0x94, 0xa0, 0x3a, 0x8c, 0xde, 0x73, 0xbc, 0xd8, 0xf3, 0xb7, 0x04, 0xbb, 0xf0, 0x4c, 0xcf, - 0x2b, 0x85, 0x55, 0xba, 0xcb, 0x2b, 0xf0, 0x4b, 0x4f, 0xfc, 0xc1, 0x92, 0x0c, 0xa5, 0x18, 0x76, - 0x7c, 0x9f, 0x52, 0x2c, 0x0c, 0x4a, 0x11, 0xf3, 0x0a, 0x9c, 0xa2, 0xf8, 0x83, 0x25, 0x19, 0xf4, - 0x36, 0x80, 0x5c, 0x96, 0xc4, 0x15, 0xd6, 0x0f, 0xcf, 0xf5, 0x27, 0xba, 0xae, 0xea, 0x2c, 0x4e, - 0xd2, 0x2b, 0x35, 0xf9, 0x8f, 0x35, 0x7a, 0x76, 0xcc, 0xd8, 0xaa, 0xee, 0xce, 0xa0, 0x6f, 0xa7, - 0x27, 0x81, 0x13, 0xc6, 0xc4, 0x5d, 0x88, 0xc5, 0xe0, 0x7c, 0x7c, 0x30, 0x99, 0x62, 0xdd, 0xdb, - 0x25, 0xfa, 0xa9, 0x21, 0x88, 0xe0, 0x84, 0x9e, 0xfd, 0x8b, 0x45, 0x98, 0xcd, 0xeb, 0x2e, 0x5d, - 0x74, 0xe4, 0xbe, 0x17, 0x2f, 0x51, 0x6e, 0xc8, 0x32, 0x17, 0xdd, 0xb2, 0x28, 0xc7, 0x0a, 0x83, - 0xce, 0x7e, 0xe4, 0x6d, 0x49, 0x91, 0x70, 0x38, 0x99, 0xfd, 0x06, 0x2b, 0xc5, 0x02, 0x4a, 0xf1, - 0x42, 0xe2, 0x44, 0xc2, 0xbc, 0x46, 0x5b, 0x25, 0x98, 0x95, 0x62, 0x01, 0xd5, 0xf5, 0x4d, 0x43, - 0x7d, 0xf4, 0x4d, 0xc6, 0x10, 0x0d, 0x1f, 0xef, 0x10, 0xa1, 0x2f, 0x00, 0x6c, 0x7a, 0xbe, 0x17, - 0x6d, 0x33, 0xea, 0x23, 0x47, 0xa6, 0xae, 0x78, 0xa9, 0x15, 0x45, 0x05, 0x6b, 0x14, 0xd1, 0xcb, - 0x30, 0xa6, 0x36, 0x60, 0xad, 0xca, 0xde, 0x1a, 0x35, 0xdb, 0x8d, 0xe4, 0x34, 0xaa, 0x62, 0x1d, - 0xcf, 0x7e, 0x27, 0xbd, 0x5e, 0xc4, 0x0e, 0xd0, 0xc6, 0xd7, 0x1a, 0x74, 0x7c, 0x0b, 0xbd, 0xc7, - 0xd7, 0xfe, 0xb5, 0x22, 0x4c, 0x19, 0x8d, 0x75, 0xa2, 0x01, 0xce, 0xac, 0x6b, 0xf4, 0x9e, 0x73, - 0x62, 0x22, 0xf6, 0x9f, 0xdd, 0x7f, 0xab, 0xe8, 0x77, 0x21, 0xdd, 0x01, 0xbc, 0x3e, 0xfa, 0x02, - 0x94, 0x5b, 0x4e, 0xc4, 0x74, 0x57, 0x44, 0xec, 0xbb, 0x41, 0x88, 0x25, 0x72, 0x84, 0x13, 0xc5, - 0xda, 0x55, 0xc3, 0x69, 0x27, 0x24, 0xe9, 0x85, 0x4c, 0x79, 0x1f, 0x69, 0xbf, 0xa5, 0x3a, 0x41, - 0x19, 0xa4, 0x7d, 0xcc, 0x61, 0xe8, 0x15, 0x18, 0x0f, 0x09, 0x5b, 0x15, 0x4b, 0x94, 0x95, 0x63, - 0xcb, 0x6c, 0x38, 0xe1, 0xf9, 0xb0, 0x06, 0xc3, 0x06, 0x66, 0xc2, 0xca, 0x8f, 0xf4, 0x60, 0xe5, - 0x9f, 0x81, 0x51, 0xf6, 0x43, 0xad, 0x00, 0x35, 0x1b, 0x35, 0x5e, 0x8c, 0x25, 0x3c, 0xbd, 0x60, - 0x4a, 0x03, 0x2e, 0x98, 0x8f, 0xc3, 0x64, 0xd5, 0x21, 0xbb, 0x81, 0xbf, 0xec, 0xbb, 0xed, 0xc0, - 0xf3, 0x63, 0x34, 0x0b, 0x43, 0xec, 0x76, 0xe0, 0x7b, 0x7b, 0x88, 0x52, 0xc0, 0x43, 0x94, 0x31, - 0xb7, 0xb7, 0xe0, 0x4c, 0x35, 0xb8, 0xe7, 0xdf, 0x73, 0x42, 0x77, 0xa1, 0x5e, 0xd3, 0xe4, 0xdc, - 0x35, 0x29, 0x67, 0x71, 0x7b, 0xa8, 0xcc, 0x33, 0x55, 0xab, 0xc9, 0xef, 0xda, 0x15, 0xaf, 0x45, - 0x72, 0xb4, 0x11, 0x7f, 0xa7, 0x60, 0xb4, 0x94, 0xe0, 0xab, 0x07, 0x23, 0x2b, 0xf7, 0xc1, 0xe8, - 0x4d, 0x28, 0x6d, 0x7a, 0xa4, 0xe5, 0x62, 0xb2, 0x29, 0x96, 0xd8, 0xd3, 0xf9, 0x26, 0x1e, 0x2b, - 0x14, 0x53, 0x6a, 0x9f, 0xb8, 0x94, 0xb6, 0x22, 0x2a, 0x63, 0x45, 0x06, 0xed, 0xc0, 0xb4, 0x14, - 0x03, 0x24, 0x54, 0x2c, 0xb8, 0x67, 0x7a, 0xc9, 0x16, 0x26, 0xf1, 0xd3, 0x0f, 0x0e, 0x2a, 0xd3, - 0x38, 0x45, 0x06, 0x77, 0x11, 0xa6, 0x62, 0xd9, 0x2e, 0x3d, 0x5a, 0x87, 0xd8, 0xf0, 0x33, 0xb1, - 0x8c, 0x49, 0x98, 0xac, 0xd4, 0xfe, 0x11, 0x0b, 0x1e, 0xef, 0x1a, 0x19, 0x21, 0x69, 0x1f, 0xf3, - 0x2c, 0xa4, 0x25, 0xdf, 0x42, 0x7f, 0xc9, 0xd7, 0xfe, 0x19, 0x0b, 0x4e, 0x2f, 0xef, 0xb6, 0xe3, - 0xfd, 0xaa, 0x67, 0xbe, 0xee, 0x7c, 0x0a, 0x46, 0x76, 0x89, 0xeb, 0x75, 0x76, 0xc5, 0xcc, 0x55, - 0xe4, 0xf1, 0xb3, 0xca, 0x4a, 0x0f, 0x0f, 0x2a, 0x13, 0x8d, 0x38, 0x08, 0x9d, 0x2d, 0xc2, 0x0b, - 0xb0, 0x40, 0x67, 0x87, 0xb8, 0xf7, 0x1e, 0xb9, 0xe9, 0xed, 0x7a, 0xd2, 0x64, 0xa7, 0xa7, 0xee, - 0x6c, 0x5e, 0x0e, 0xe8, 0xfc, 0x9b, 0x1d, 0xc7, 0x8f, 0xbd, 0x78, 0x5f, 0x3c, 0xcc, 0x48, 0x22, - 0x38, 0xa1, 0x67, 0x7f, 0xc3, 0x82, 0x29, 0xb9, 0xee, 0x17, 0x5c, 0x37, 0x24, 0x51, 0x84, 0xe6, - 0xa0, 0xe0, 0xb5, 0x45, 0x2f, 0x41, 0xf4, 0xb2, 0x50, 0xab, 0xe3, 0x82, 0xd7, 0x46, 0x75, 0x28, - 0x73, 0xcb, 0x9f, 0x64, 0x71, 0x0d, 0x64, 0x3f, 0xc4, 0x7a, 0xb0, 0x2e, 0x6b, 0xe2, 0x84, 0x88, - 0xe4, 0xe0, 0xd8, 0x99, 0x59, 0x34, 0x5f, 0xbd, 0xae, 0x8b, 0x72, 0xac, 0x30, 0xd0, 0x65, 0x28, - 0xf9, 0x81, 0xcb, 0x0d, 0xb1, 0xf8, 0xed, 0xc7, 0x96, 0xec, 0x9a, 0x28, 0xc3, 0x0a, 0x6a, 0xff, - 0xa0, 0x05, 0xe3, 0xf2, 0xcb, 0x06, 0x64, 0x26, 0xe9, 0xd6, 0x4a, 0x18, 0xc9, 0x64, 0x6b, 0x51, - 0x66, 0x90, 0x41, 0x0c, 0x1e, 0xb0, 0x78, 0x14, 0x1e, 0xd0, 0xfe, 0xe1, 0x02, 0x4c, 0xca, 0xee, - 0x34, 0x3a, 0x1b, 0x11, 0x89, 0xd1, 0x3a, 0x94, 0x1d, 0x3e, 0xe4, 0x44, 0xae, 0xd8, 0xa7, 0xb2, - 0x85, 0x0f, 0x63, 0x7e, 0x92, 0x6b, 0x79, 0x41, 0xd6, 0xc6, 0x09, 0x21, 0xd4, 0x82, 0x19, 0x3f, - 0x88, 0xd9, 0x11, 0xad, 0xe0, 0xbd, 0x9e, 0x40, 0xd2, 0xd4, 0xcf, 0x0a, 0xea, 0x33, 0x6b, 0x69, - 0x2a, 0xb8, 0x9b, 0x30, 0x5a, 0x96, 0x0a, 0x8f, 0x62, 0xbe, 0xb8, 0xa1, 0xcf, 0x42, 0xb6, 0xbe, - 0xc3, 0xfe, 0x15, 0x0b, 0xca, 0x12, 0xed, 0x24, 0x5e, 0xbb, 0x56, 0x61, 0x34, 0x62, 0x93, 0x20, - 0x87, 0xc6, 0xee, 0xd5, 0x71, 0x3e, 0x5f, 0xc9, 0xcd, 0xc3, 0xff, 0x47, 0x58, 0xd2, 0x60, 0xfa, - 0x6e, 0xd5, 0xfd, 0x0f, 0x89, 0xbe, 0x5b, 0xf5, 0x27, 0xe7, 0x86, 0xf9, 0x7d, 0xd6, 0x67, 0x4d, - 0xac, 0xa5, 0x0c, 0x52, 0x3b, 0x24, 0x9b, 0xde, 0xfd, 0x34, 0x83, 0x54, 0x67, 0xa5, 0x58, 0x40, - 0xd1, 0xdb, 0x30, 0xde, 0x94, 0x8a, 0xce, 0xe4, 0x18, 0xb8, 0xd4, 0x53, 0xe9, 0xae, 0xde, 0x67, - 0xb8, 0x91, 0xf6, 0x92, 0x56, 0x1f, 0x1b, 0xd4, 0xcc, 0xe7, 0xf6, 0x62, 0xbf, 0xe7, 0xf6, 0x84, - 0x6e, 0xfe, 0xe3, 0xf3, 0x8f, 0x5a, 0x30, 0xc2, 0xd5, 0x65, 0x83, 0xe9, 0x17, 0xb5, 0xe7, 0xaa, - 0x64, 0xec, 0xee, 0xd0, 0x42, 0xf1, 0xfc, 0x84, 0x56, 0xa1, 0xcc, 0x7e, 0x30, 0xb5, 0x41, 0x31, - 0xdf, 0x3a, 0x9d, 0xb7, 0xaa, 0x77, 0xf0, 0x8e, 0xac, 0x86, 0x13, 0x0a, 0xf6, 0x57, 0x8b, 0xf4, - 0xa8, 0x4a, 0x50, 0x8d, 0x1b, 0xdc, 0x7a, 0x74, 0x37, 0x78, 0xe1, 0x51, 0xdd, 0xe0, 0x5b, 0x30, - 0xd5, 0xd4, 0x1e, 0xb7, 0x92, 0x99, 0xbc, 0xdc, 0x73, 0x91, 0x68, 0xef, 0x60, 0x5c, 0x65, 0xb4, - 0x64, 0x12, 0xc1, 0x69, 0xaa, 0xe8, 0xdb, 0x61, 0x9c, 0xcf, 0xb3, 0x68, 0x85, 0x5b, 0x2c, 0x7c, - 0x2c, 0x7f, 0xbd, 0xe8, 0x4d, 0xb0, 0x95, 0xd8, 0xd0, 0xaa, 0x63, 0x83, 0x98, 0xfd, 0x8b, 0x25, - 0x18, 0x5e, 0xde, 0x23, 0x7e, 0x7c, 0x02, 0x07, 0x52, 0x13, 0x26, 0x3d, 0x7f, 0x2f, 0x68, 0xed, - 0x11, 0x97, 0xc3, 0x8f, 0x72, 0xb9, 0x3e, 0x26, 0x48, 0x4f, 0xd6, 0x0c, 0x12, 0x38, 0x45, 0xf2, - 0x51, 0x48, 0x98, 0xd7, 0x60, 0x84, 0xcf, 0xbd, 0x10, 0x2f, 0x33, 0x95, 0xc1, 0x6c, 0x10, 0xc5, - 0x2e, 0x48, 0xa4, 0x5f, 0xae, 0x7d, 0x16, 0xd5, 0xd1, 0x3b, 0x30, 0xb9, 0xe9, 0x85, 0x51, 0x4c, - 0x45, 0xc3, 0x28, 0x76, 0x76, 0xdb, 0x0f, 0x21, 0x51, 0xaa, 0x71, 0x58, 0x31, 0x28, 0xe1, 0x14, - 0x65, 0xb4, 0x05, 0x13, 0x54, 0xc8, 0x49, 0x9a, 0x1a, 0x3d, 0x72, 0x53, 0x4a, 0x65, 0x74, 0x53, - 0x27, 0x84, 0x4d, 0xba, 0xf4, 0x30, 0x69, 0x32, 0xa1, 0xa8, 0xc4, 0x38, 0x0a, 0x75, 0x98, 0x70, - 0x69, 0x88, 0xc3, 0xe8, 0x99, 0xc4, 0xcc, 0x56, 0xca, 0xe6, 0x99, 0xa4, 0x19, 0xa7, 0x7c, 0x11, - 0xca, 0x84, 0x0e, 0x21, 0x25, 0x2c, 0x14, 0xe3, 0x57, 0x06, 0xeb, 0xeb, 0xaa, 0xd7, 0x0c, 0x03, - 0x53, 0x96, 0x5f, 0x96, 0x94, 0x70, 0x42, 0x14, 0x2d, 0xc1, 0x48, 0x44, 0x42, 0x8f, 0x44, 0x42, - 0x45, 0xde, 0x63, 0x1a, 0x19, 0x1a, 0xb7, 0xf8, 0xe4, 0xbf, 0xb1, 0xa8, 0x4a, 0x97, 0x97, 0xc3, - 0xa4, 0x21, 0xa6, 0x15, 0xd7, 0x96, 0xd7, 0x02, 0x2b, 0xc5, 0x02, 0x8a, 0xde, 0x80, 0xd1, 0x90, - 0xb4, 0x98, 0xb2, 0x68, 0x62, 0xf0, 0x45, 0xce, 0x75, 0x4f, 0xbc, 0x1e, 0x96, 0x04, 0xd0, 0x0d, - 0x40, 0x21, 0xa1, 0x3c, 0x84, 0xe7, 0x6f, 0x29, 0x63, 0x0e, 0xa1, 0xeb, 0x7e, 0x42, 0xb4, 0x7f, - 0x0a, 0x27, 0x18, 0xd2, 0xf8, 0x16, 0x67, 0x54, 0x43, 0xd7, 0x60, 0x46, 0x95, 0xd6, 0xfc, 0x28, - 0x76, 0xfc, 0x26, 0x61, 0x6a, 0xee, 0x72, 0xc2, 0x15, 0xe1, 0x34, 0x02, 0xee, 0xae, 0x63, 0xff, - 0x14, 0x65, 0x67, 0xe8, 0x68, 0x9d, 0x00, 0x2f, 0xf0, 0xba, 0xc9, 0x0b, 0x9c, 0xcd, 0x9d, 0xb9, - 0x1c, 0x3e, 0xe0, 0x81, 0x05, 0x63, 0xda, 0xcc, 0x26, 0x6b, 0xd6, 0xea, 0xb1, 0x66, 0x3b, 0x30, - 0x4d, 0x57, 0xfa, 0xad, 0x8d, 0x88, 0x84, 0x7b, 0xc4, 0x65, 0x0b, 0xb3, 0xf0, 0x70, 0x0b, 0x53, - 0xbd, 0x32, 0xdf, 0x4c, 0x11, 0xc4, 0x5d, 0x4d, 0xa0, 0x4f, 0x49, 0xcd, 0x49, 0xd1, 0x30, 0xd2, - 0xe2, 0x5a, 0x91, 0xc3, 0x83, 0xca, 0xb4, 0xf6, 0x21, 0xba, 0xa6, 0xc4, 0xfe, 0xa2, 0xfc, 0x46, - 0xf5, 0x9a, 0xdf, 0x54, 0x8b, 0x25, 0xf5, 0x9a, 0xaf, 0x96, 0x03, 0x4e, 0x70, 0xe8, 0x1e, 0xa5, - 0x22, 0x48, 0xfa, 0x35, 0x9f, 0x0a, 0x28, 0x98, 0x41, 0xec, 0x17, 0x01, 0x96, 0xef, 0x93, 0x26, - 0x5f, 0xea, 0xfa, 0x03, 0xa4, 0x95, 0xff, 0x00, 0x69, 0xff, 0x67, 0x0b, 0x26, 0x57, 0x96, 0x0c, - 0x31, 0x71, 0x1e, 0x80, 0xcb, 0x46, 0x77, 0xef, 0xae, 0x49, 0xdd, 0x3a, 0x57, 0x8f, 0xaa, 0x52, - 0xac, 0x61, 0xa0, 0xb3, 0x50, 0x6c, 0x75, 0x7c, 0x21, 0xb2, 0x8c, 0x3e, 0x38, 0xa8, 0x14, 0x6f, - 0x76, 0x7c, 0x4c, 0xcb, 0x34, 0x0b, 0xc1, 0xe2, 0xc0, 0x16, 0x82, 0x7d, 0x3d, 0xf5, 0x50, 0x05, - 0x86, 0xef, 0xdd, 0xf3, 0x5c, 0xee, 0x0f, 0x21, 0xf4, 0xfe, 0x77, 0xef, 0xd6, 0xaa, 0x11, 0xe6, - 0xe5, 0xf6, 0x57, 0x8a, 0x30, 0xb7, 0xd2, 0x22, 0xf7, 0xdf, 0xa7, 0x4f, 0xc8, 0xa0, 0xf6, 0x8d, - 0x47, 0xe3, 0x17, 0x8f, 0x6a, 0xc3, 0xda, 0x7f, 0x3c, 0x36, 0x61, 0x94, 0x3f, 0x66, 0x4b, 0x0f, - 0x91, 0x57, 0xb3, 0x5a, 0xcf, 0x1f, 0x90, 0x79, 0xfe, 0x28, 0x2e, 0x0c, 0xdc, 0xd5, 0x4d, 0x2b, - 0x4a, 0xb1, 0x24, 0x3e, 0xf7, 0x69, 0x18, 0xd7, 0x31, 0x8f, 0x64, 0x4d, 0xfe, 0x57, 0x8a, 0x30, - 0x4d, 0x7b, 0xf0, 0x48, 0x27, 0xe2, 0x76, 0xf7, 0x44, 0x1c, 0xb7, 0x45, 0x71, 0xff, 0xd9, 0x78, - 0x3b, 0x3d, 0x1b, 0x2f, 0xe4, 0xcd, 0xc6, 0x49, 0xcf, 0xc1, 0xf7, 0x58, 0x70, 0x6a, 0xa5, 0x15, - 0x34, 0x77, 0x52, 0x56, 0xbf, 0x2f, 0xc3, 0x18, 0x3d, 0xc7, 0x23, 0xc3, 0x21, 0xcd, 0x70, 0x51, - 0x14, 0x20, 0xac, 0xe3, 0x69, 0xd5, 0x6e, 0xdf, 0xae, 0x55, 0xb3, 0x3c, 0x1b, 0x05, 0x08, 0xeb, - 0x78, 0xf6, 0xd7, 0x2d, 0x38, 0x7f, 0x6d, 0x69, 0x39, 0x59, 0x8a, 0x5d, 0xce, 0x95, 0x54, 0x0a, - 0x74, 0xb5, 0xae, 0x24, 0x52, 0x60, 0x95, 0xf5, 0x42, 0x40, 0x3f, 0x2c, 0x8e, 0xc3, 0x3f, 0x69, - 0xc1, 0xa9, 0x6b, 0x5e, 0x4c, 0xaf, 0xe5, 0xb4, 0x9b, 0x1f, 0xbd, 0x97, 0x23, 0x2f, 0x0e, 0xc2, - 0xfd, 0xb4, 0x9b, 0x1f, 0x56, 0x10, 0xac, 0x61, 0xf1, 0x96, 0xf7, 0x3c, 0x66, 0x46, 0x55, 0x30, - 0x55, 0x51, 0x58, 0x94, 0x63, 0x85, 0x41, 0x3f, 0xcc, 0xf5, 0x42, 0x26, 0x4a, 0xec, 0x8b, 0x13, - 0x56, 0x7d, 0x58, 0x55, 0x02, 0x70, 0x82, 0x63, 0xff, 0xa1, 0x05, 0x95, 0x6b, 0xad, 0x4e, 0x14, - 0x93, 0x70, 0x33, 0xca, 0x39, 0x1d, 0x5f, 0x84, 0x32, 0x91, 0x82, 0xbb, 0xe8, 0xb5, 0x62, 0x35, - 0x95, 0x44, 0xcf, 0xbd, 0x0d, 0x15, 0xde, 0x00, 0x3e, 0x04, 0x47, 0x33, 0x02, 0x5f, 0x01, 0x44, - 0xf4, 0xb6, 0x74, 0xf7, 0x4b, 0xe6, 0xc7, 0xb5, 0xdc, 0x05, 0xc5, 0x19, 0x35, 0xec, 0x1f, 0xb1, - 0xe0, 0x8c, 0xfa, 0xe0, 0x0f, 0xdd, 0x67, 0xda, 0x3f, 0x57, 0x80, 0x89, 0xeb, 0xeb, 0xeb, 0xf5, - 0x6b, 0x24, 0x16, 0xd7, 0x76, 0x7f, 0xdd, 0x3a, 0xd6, 0x54, 0x84, 0xbd, 0xa4, 0xc0, 0x4e, 0xec, - 0xb5, 0xe6, 0xb9, 0x17, 0xff, 0x7c, 0xcd, 0x8f, 0x6f, 0x85, 0x8d, 0x38, 0xf4, 0xfc, 0xad, 0x4c, - 0xa5, 0xa2, 0x64, 0x2e, 0x8a, 0x79, 0xcc, 0x05, 0x7a, 0x11, 0x46, 0x58, 0x18, 0x01, 0x39, 0x09, - 0x4f, 0x28, 0x21, 0x8a, 0x95, 0x1e, 0x1e, 0x54, 0xca, 0xb7, 0x71, 0x8d, 0xff, 0xc1, 0x02, 0x15, - 0xdd, 0x86, 0xb1, 0xed, 0x38, 0x6e, 0x5f, 0x27, 0x8e, 0x4b, 0x42, 0x79, 0x1c, 0x5e, 0xc8, 0x3a, - 0x0e, 0xe9, 0x20, 0x70, 0xb4, 0xe4, 0x04, 0x49, 0xca, 0x22, 0xac, 0xd3, 0xb1, 0x1b, 0x00, 0x09, - 0xec, 0x98, 0x14, 0x2a, 0xf6, 0xef, 0x59, 0x30, 0xca, 0x3d, 0x3a, 0x43, 0xf4, 0x1a, 0x0c, 0x91, - 0xfb, 0xa4, 0x29, 0x58, 0xe5, 0xcc, 0x0e, 0x27, 0x9c, 0x16, 0x7f, 0x1e, 0xa0, 0xff, 0x31, 0xab, - 0x85, 0xae, 0xc3, 0x28, 0xed, 0xed, 0x35, 0xe5, 0xde, 0xfa, 0x64, 0xde, 0x17, 0xab, 0x69, 0xe7, - 0xcc, 0x99, 0x28, 0xc2, 0xb2, 0x3a, 0x53, 0x75, 0x37, 0xdb, 0x0d, 0x7a, 0x62, 0xc7, 0xbd, 0x18, - 0x8b, 0xf5, 0xa5, 0x3a, 0x47, 0x12, 0xd4, 0xb8, 0xaa, 0x5b, 0x16, 0xe2, 0x84, 0x88, 0xbd, 0x0e, - 0x65, 0x3a, 0xa9, 0x0b, 0x2d, 0xcf, 0xe9, 0xad, 0x65, 0x7f, 0x16, 0xca, 0x52, 0xe3, 0x1d, 0x09, - 0x4f, 0x2e, 0x46, 0x55, 0x2a, 0xc4, 0x23, 0x9c, 0xc0, 0xed, 0x4d, 0x38, 0xcd, 0x4c, 0x1d, 0x9c, - 0x78, 0xdb, 0xd8, 0x63, 0xfd, 0x17, 0xf3, 0x73, 0x42, 0xf2, 0xe4, 0x33, 0x33, 0xab, 0x39, 0x4b, - 0x8c, 0x4b, 0x8a, 0x89, 0x14, 0x6a, 0xff, 0xc1, 0x10, 0x3c, 0x51, 0x6b, 0xe4, 0x3b, 0xfb, 0xbe, - 0x02, 0xe3, 0x9c, 0x2f, 0xa5, 0x4b, 0xdb, 0x69, 0x89, 0x76, 0xd5, 0x43, 0xe0, 0xba, 0x06, 0xc3, - 0x06, 0x26, 0x3a, 0x0f, 0x45, 0xef, 0x5d, 0x3f, 0x6d, 0x77, 0x5c, 0x7b, 0x73, 0x0d, 0xd3, 0x72, - 0x0a, 0xa6, 0x2c, 0x2e, 0xbf, 0x3b, 0x14, 0x58, 0xb1, 0xb9, 0xaf, 0xc3, 0xa4, 0x17, 0x35, 0x23, - 0xaf, 0xe6, 0xd3, 0x73, 0x46, 0x3b, 0xa9, 0x94, 0x56, 0x84, 0x76, 0x5a, 0x41, 0x71, 0x0a, 0x5b, - 0xbb, 0xc8, 0x86, 0x07, 0x66, 0x93, 0xfb, 0xba, 0x36, 0x51, 0x09, 0xa0, 0xcd, 0xbe, 0x2e, 0x62, - 0x56, 0x7c, 0x42, 0x02, 0xe0, 0x1f, 0x1c, 0x61, 0x09, 0xa3, 0x22, 0x67, 0x73, 0xdb, 0x69, 0x2f, - 0x74, 0xe2, 0xed, 0xaa, 0x17, 0x35, 0x83, 0x3d, 0x12, 0xee, 0x33, 0x6d, 0x41, 0x29, 0x11, 0x39, - 0x15, 0x60, 0xe9, 0xfa, 0x42, 0x9d, 0x62, 0xe2, 0xee, 0x3a, 0x26, 0x1b, 0x0c, 0xc7, 0xc1, 0x06, - 0x2f, 0xc0, 0x94, 0x6c, 0xa6, 0x41, 0x22, 0x76, 0x29, 0x8e, 0xb1, 0x8e, 0x29, 0xdb, 0x62, 0x51, - 0xac, 0xba, 0x95, 0xc6, 0x47, 0x9f, 0x82, 0x09, 0xcf, 0xf7, 0x62, 0xcf, 0x89, 0x83, 0x90, 0xb1, - 0x14, 0x5c, 0x31, 0xc0, 0x4c, 0xf7, 0x6a, 0x3a, 0x00, 0x9b, 0x78, 0xf6, 0x7f, 0x1f, 0x82, 0x19, - 0x36, 0x6d, 0xdf, 0x5c, 0x61, 0x1f, 0x9a, 0x15, 0x76, 0xbb, 0x7b, 0x85, 0x1d, 0x07, 0x7f, 0xff, - 0x41, 0x2e, 0xb3, 0x77, 0xa0, 0xac, 0x8c, 0x9f, 0xa5, 0xf7, 0x83, 0x95, 0xe3, 0xfd, 0xd0, 0x9f, - 0xfb, 0x90, 0xef, 0xd6, 0xc5, 0xcc, 0x77, 0xeb, 0xbf, 0x67, 0x41, 0x62, 0x03, 0x8a, 0xae, 0x43, - 0xb9, 0x1d, 0x30, 0x3b, 0x8b, 0x50, 0x1a, 0x2f, 0x3d, 0x91, 0x79, 0x51, 0xf1, 0x4b, 0x91, 0x8f, - 0x5f, 0x5d, 0xd6, 0xc0, 0x49, 0x65, 0xb4, 0x08, 0xa3, 0xed, 0x90, 0x34, 0x62, 0xe6, 0xf3, 0xdb, - 0x97, 0x0e, 0x5f, 0x23, 0x1c, 0x1f, 0xcb, 0x8a, 0xf6, 0xcf, 0x5b, 0x00, 0xfc, 0x69, 0xd8, 0xf1, - 0xb7, 0xc8, 0x09, 0xa8, 0xbb, 0xab, 0x30, 0x14, 0xb5, 0x49, 0xb3, 0x97, 0x05, 0x4c, 0xd2, 0x9f, - 0x46, 0x9b, 0x34, 0x93, 0x01, 0xa7, 0xff, 0x30, 0xab, 0x6d, 0x7f, 0x2f, 0xc0, 0x64, 0x82, 0x56, - 0x8b, 0xc9, 0x2e, 0x7a, 0xde, 0xf0, 0x01, 0x3c, 0x9b, 0xf2, 0x01, 0x2c, 0x33, 0x6c, 0x4d, 0xb3, - 0xfa, 0x0e, 0x14, 0x77, 0x9d, 0xfb, 0x42, 0x75, 0xf6, 0x6c, 0xef, 0x6e, 0x50, 0xfa, 0xf3, 0xab, - 0xce, 0x7d, 0x2e, 0x24, 0x3e, 0x2b, 0x17, 0xc8, 0xaa, 0x73, 0xff, 0x90, 0xdb, 0xb9, 0xb0, 0x43, - 0xea, 0xa6, 0x17, 0xc5, 0x5f, 0xfa, 0x6f, 0xc9, 0x7f, 0xb6, 0xec, 0x68, 0x23, 0xac, 0x2d, 0xcf, - 0x17, 0x0f, 0xa5, 0x03, 0xb5, 0xe5, 0xf9, 0xe9, 0xb6, 0x3c, 0x7f, 0x80, 0xb6, 0x3c, 0x1f, 0xbd, - 0x07, 0xa3, 0xc2, 0x28, 0x41, 0xf8, 0xdc, 0x5f, 0x19, 0xa0, 0x3d, 0x61, 0xd3, 0xc0, 0xdb, 0xbc, - 0x22, 0x85, 0x60, 0x51, 0xda, 0xb7, 0x5d, 0xd9, 0x20, 0xfa, 0xdb, 0x16, 0x4c, 0x8a, 0xdf, 0x98, - 0xbc, 0xdb, 0x21, 0x51, 0x2c, 0x78, 0xcf, 0x4f, 0x0e, 0xde, 0x07, 0x51, 0x91, 0x77, 0xe5, 0x93, - 0xf2, 0x98, 0x35, 0x81, 0x7d, 0x7b, 0x94, 0xea, 0x05, 0xfa, 0x27, 0x16, 0x9c, 0xde, 0x75, 0xee, - 0xf3, 0x16, 0x79, 0x19, 0x76, 0x62, 0x2f, 0x10, 0xc6, 0xfa, 0xaf, 0x0d, 0x36, 0xfd, 0x5d, 0xd5, - 0x79, 0x27, 0xa5, 0x5d, 0xef, 0xe9, 0x2c, 0x94, 0xbe, 0x5d, 0xcd, 0xec, 0xd7, 0xdc, 0x26, 0x94, - 0xe4, 0x7a, 0xcb, 0x50, 0x35, 0x54, 0x75, 0xc6, 0xfa, 0xc8, 0x36, 0x21, 0xba, 0x23, 0x1e, 0x6d, - 0x47, 0xac, 0xb5, 0x47, 0xda, 0xce, 0x3b, 0x30, 0xae, 0xaf, 0xb1, 0x47, 0xda, 0xd6, 0xbb, 0x70, - 0x2a, 0x63, 0x2d, 0x3d, 0xd2, 0x26, 0xef, 0xc1, 0xd9, 0xdc, 0xf5, 0xf1, 0x28, 0x1b, 0xb6, 0x7f, - 0xce, 0xd2, 0xcf, 0xc1, 0x13, 0x78, 0x73, 0x58, 0x32, 0xdf, 0x1c, 0x2e, 0xf4, 0xde, 0x39, 0x39, - 0x0f, 0x0f, 0x6f, 0xeb, 0x9d, 0xa6, 0xa7, 0x3a, 0x7a, 0x03, 0x46, 0x5a, 0xb4, 0x44, 0x5a, 0xc3, - 0xd8, 0xfd, 0x77, 0x64, 0xc2, 0x4b, 0xb1, 0xf2, 0x08, 0x0b, 0x0a, 0xf6, 0x2f, 0x59, 0x30, 0x74, - 0x02, 0x23, 0x81, 0xcd, 0x91, 0x78, 0x3e, 0x97, 0xb4, 0x08, 0x07, 0x38, 0x8f, 0x9d, 0x7b, 0xcb, - 0xf7, 0x63, 0xe2, 0x47, 0x4c, 0x54, 0xcc, 0x1c, 0x98, 0xef, 0x80, 0x53, 0x37, 0x03, 0xc7, 0x5d, - 0x74, 0x5a, 0x8e, 0xdf, 0x24, 0x61, 0xcd, 0xdf, 0xea, 0x6b, 0x96, 0xa5, 0x1b, 0x51, 0x15, 0xfa, - 0x19, 0x51, 0xd9, 0xdb, 0x80, 0xf4, 0x06, 0x84, 0xe1, 0x2a, 0x86, 0x51, 0x8f, 0x37, 0x25, 0x86, - 0xff, 0xe9, 0x6c, 0xee, 0xae, 0xab, 0x67, 0x9a, 0x49, 0x26, 0x2f, 0xc0, 0x92, 0x90, 0xfd, 0x0a, - 0x64, 0x3a, 0xab, 0xf5, 0x57, 0x1b, 0xd8, 0x9f, 0x83, 0x19, 0x56, 0xf3, 0x88, 0x22, 0xad, 0x9d, - 0xd2, 0x4a, 0x66, 0x44, 0xa6, 0xb1, 0xbf, 0x6c, 0xc1, 0xd4, 0x5a, 0x2a, 0x60, 0xc7, 0x25, 0xf6, - 0x00, 0x9a, 0xa1, 0x0c, 0x6f, 0xb0, 0x52, 0x2c, 0xa0, 0xc7, 0xae, 0x83, 0xfa, 0x33, 0x0b, 0x12, - 0xff, 0xd1, 0x13, 0x60, 0xbc, 0x96, 0x0c, 0xc6, 0x2b, 0x53, 0x37, 0xa2, 0xba, 0x93, 0xc7, 0x77, - 0xa1, 0x1b, 0x2a, 0x58, 0x42, 0x0f, 0xb5, 0x48, 0x42, 0x86, 0xbb, 0xd6, 0x4f, 0x9a, 0x11, 0x15, - 0x64, 0xf8, 0x04, 0x66, 0x3b, 0xa5, 0x70, 0x3f, 0x24, 0xb6, 0x53, 0xaa, 0x3f, 0x39, 0x3b, 0xb4, - 0xae, 0x75, 0x99, 0x9d, 0x5c, 0xdf, 0xca, 0x6c, 0xe1, 0x9d, 0x96, 0xf7, 0x1e, 0x51, 0x11, 0x5f, - 0x2a, 0xc2, 0xb6, 0x5d, 0x94, 0x1e, 0x1e, 0x54, 0x26, 0xd4, 0x3f, 0x1e, 0x61, 0x2e, 0xa9, 0x62, - 0x5f, 0x87, 0xa9, 0xd4, 0x80, 0xa1, 0x97, 0x61, 0xb8, 0xbd, 0xed, 0x44, 0x24, 0x65, 0x2f, 0x3a, - 0x5c, 0xa7, 0x85, 0x87, 0x07, 0x95, 0x49, 0x55, 0x81, 0x95, 0x60, 0x8e, 0x6d, 0xff, 0x2f, 0x0b, - 0x86, 0xd6, 0x02, 0xf7, 0x24, 0x16, 0xd3, 0xeb, 0xc6, 0x62, 0x3a, 0x97, 0x17, 0x9f, 0x33, 0x77, - 0x1d, 0xad, 0xa4, 0xd6, 0xd1, 0x85, 0x5c, 0x0a, 0xbd, 0x97, 0xd0, 0x2e, 0x8c, 0xb1, 0xa8, 0x9f, - 0xc2, 0x7e, 0xf5, 0x45, 0x43, 0x06, 0xa8, 0xa4, 0x64, 0x80, 0x29, 0x0d, 0x55, 0x93, 0x04, 0x9e, - 0x81, 0x51, 0x61, 0x43, 0x99, 0xb6, 0xfa, 0x17, 0xb8, 0x58, 0xc2, 0xed, 0x1f, 0x2d, 0x82, 0x11, - 0x65, 0x14, 0xfd, 0x8a, 0x05, 0xf3, 0x21, 0x77, 0xa3, 0x74, 0xab, 0x9d, 0xd0, 0xf3, 0xb7, 0x1a, - 0xcd, 0x6d, 0xe2, 0x76, 0x5a, 0x9e, 0xbf, 0x55, 0xdb, 0xf2, 0x03, 0x55, 0xbc, 0x7c, 0x9f, 0x34, - 0x3b, 0xec, 0x21, 0xa4, 0x4f, 0x48, 0x53, 0x65, 0xa3, 0x74, 0xf5, 0xc1, 0x41, 0x65, 0x1e, 0x1f, - 0x89, 0x36, 0x3e, 0x62, 0x5f, 0xd0, 0xd7, 0x2d, 0xb8, 0xc2, 0x83, 0x6f, 0x0e, 0xde, 0xff, 0x1e, - 0x12, 0x53, 0x5d, 0x92, 0x4a, 0x88, 0xac, 0x93, 0x70, 0x77, 0xf1, 0x53, 0x62, 0x40, 0xaf, 0xd4, - 0x8f, 0xd6, 0x16, 0x3e, 0x6a, 0xe7, 0xec, 0x7f, 0x53, 0x84, 0x09, 0xe1, 0xc1, 0x2f, 0x42, 0xc3, - 0xbc, 0x6c, 0x2c, 0x89, 0x27, 0x53, 0x4b, 0x62, 0xc6, 0x40, 0x3e, 0x9e, 0xa8, 0x30, 0x11, 0xcc, - 0xb4, 0x9c, 0x28, 0xbe, 0x4e, 0x9c, 0x30, 0xde, 0x20, 0x0e, 0xb7, 0xdd, 0x29, 0x1e, 0xd9, 0xce, - 0x48, 0xa9, 0x68, 0x6e, 0xa6, 0x89, 0xe1, 0x6e, 0xfa, 0x68, 0x0f, 0x10, 0x33, 0x40, 0x0a, 0x1d, - 0x3f, 0xe2, 0xdf, 0xe2, 0x89, 0x37, 0x83, 0xa3, 0xb5, 0x3a, 0x27, 0x5a, 0x45, 0x37, 0xbb, 0xa8, - 0xe1, 0x8c, 0x16, 0x34, 0xc3, 0xb2, 0xe1, 0x41, 0x0d, 0xcb, 0x46, 0xfa, 0xb8, 0xd6, 0xf8, 0x30, - 0xdd, 0x15, 0x84, 0xe1, 0x2d, 0x28, 0x2b, 0x03, 0x40, 0x71, 0xe8, 0xf4, 0x8e, 0x65, 0x92, 0xa6, - 0xc0, 0xd5, 0x28, 0x89, 0xf1, 0x69, 0x42, 0xce, 0xfe, 0xa7, 0x05, 0xa3, 0x41, 0x3e, 0x89, 0x6b, - 0x50, 0x72, 0xa2, 0xc8, 0xdb, 0xf2, 0x89, 0x2b, 0x76, 0xec, 0x47, 0xf3, 0x76, 0xac, 0xd1, 0x0c, - 0x33, 0xc2, 0x5c, 0x10, 0x35, 0xb1, 0xa2, 0x81, 0xae, 0x73, 0x0b, 0xa9, 0x3d, 0xc9, 0xf3, 0x0f, - 0x46, 0x0d, 0xa4, 0x0d, 0xd5, 0x1e, 0xc1, 0xa2, 0x3e, 0xfa, 0x3c, 0x37, 0x61, 0xbb, 0xe1, 0x07, - 0xf7, 0xfc, 0x6b, 0x41, 0x20, 0xdd, 0xee, 0x06, 0x23, 0x38, 0x23, 0x0d, 0xd7, 0x54, 0x75, 0x6c, - 0x52, 0x1b, 0x2c, 0x50, 0xd1, 0x77, 0xc2, 0x29, 0x4a, 0xda, 0x74, 0x9e, 0x89, 0x10, 0x81, 0x29, - 0x11, 0x1e, 0x42, 0x96, 0x89, 0xb1, 0xcb, 0x64, 0xe7, 0xcd, 0xda, 0x89, 0xd2, 0xef, 0x86, 0x49, - 0x02, 0xa7, 0x69, 0xda, 0x3f, 0x61, 0x01, 0x33, 0xfb, 0x3f, 0x01, 0x96, 0xe1, 0x33, 0x26, 0xcb, - 0x30, 0x9b, 0x37, 0xc8, 0x39, 0xdc, 0xc2, 0x4b, 0x7c, 0x65, 0xd5, 0xc3, 0xe0, 0xfe, 0xbe, 0x30, - 0x1f, 0xe8, 0xcf, 0xc9, 0xda, 0xff, 0xd7, 0xe2, 0x87, 0x98, 0xf2, 0xc4, 0x47, 0xdf, 0x05, 0xa5, - 0xa6, 0xd3, 0x76, 0x9a, 0x3c, 0x24, 0x76, 0xae, 0x56, 0xc7, 0xa8, 0x34, 0xbf, 0x24, 0x6a, 0x70, - 0x2d, 0x85, 0x0c, 0x33, 0x52, 0x92, 0xc5, 0x7d, 0x35, 0x13, 0xaa, 0xc9, 0xb9, 0x1d, 0x98, 0x30, - 0x88, 0x3d, 0x52, 0x91, 0xf6, 0xbb, 0xf8, 0x15, 0xab, 0xc2, 0xe2, 0xec, 0xc2, 0x8c, 0xaf, 0xfd, - 0xa7, 0x17, 0x8a, 0x14, 0x53, 0x3e, 0xda, 0xef, 0x12, 0x65, 0xb7, 0x8f, 0xe6, 0xd6, 0x90, 0x22, - 0x83, 0xbb, 0x29, 0xdb, 0x3f, 0x66, 0xc1, 0xe3, 0x3a, 0xa2, 0x16, 0x24, 0xa1, 0x9f, 0x9e, 0xb8, - 0x0a, 0xa5, 0xa0, 0x4d, 0x42, 0x27, 0x0e, 0x42, 0x71, 0x6b, 0x5c, 0x96, 0x83, 0x7e, 0x4b, 0x94, - 0x1f, 0x8a, 0x80, 0x92, 0x92, 0xba, 0x2c, 0xc7, 0xaa, 0x26, 0x95, 0x63, 0xd8, 0x60, 0x44, 0x22, - 0x80, 0x05, 0x3b, 0x03, 0xd8, 0x93, 0x69, 0x84, 0x05, 0xc4, 0xfe, 0x03, 0x8b, 0x2f, 0x2c, 0xbd, - 0xeb, 0xe8, 0x5d, 0x98, 0xde, 0x75, 0xe2, 0xe6, 0xf6, 0xf2, 0xfd, 0x76, 0xc8, 0xd5, 0xe3, 0x72, - 0x9c, 0x9e, 0xed, 0x37, 0x4e, 0xda, 0x47, 0x26, 0x56, 0x79, 0xab, 0x29, 0x62, 0xb8, 0x8b, 0x3c, - 0xda, 0x80, 0x31, 0x56, 0xc6, 0xcc, 0xbf, 0xa3, 0x5e, 0xac, 0x41, 0x5e, 0x6b, 0xea, 0xd5, 0x79, - 0x35, 0xa1, 0x83, 0x75, 0xa2, 0xf6, 0x97, 0x8a, 0x7c, 0xb7, 0x33, 0x6e, 0xfb, 0x19, 0x18, 0x6d, - 0x07, 0xee, 0x52, 0xad, 0x8a, 0xc5, 0x2c, 0xa8, 0x6b, 0xa4, 0xce, 0x8b, 0xb1, 0x84, 0xa3, 0x57, - 0x01, 0xc8, 0xfd, 0x98, 0x84, 0xbe, 0xd3, 0x52, 0x56, 0x32, 0xca, 0x2e, 0xb4, 0x1a, 0xac, 0x05, - 0xf1, 0xed, 0x88, 0x7c, 0xc7, 0xb2, 0x42, 0xc1, 0x1a, 0x3a, 0xba, 0x0a, 0xd0, 0x0e, 0x83, 0x3d, - 0xcf, 0x65, 0xfe, 0x84, 0x45, 0xd3, 0x86, 0xa4, 0xae, 0x20, 0x58, 0xc3, 0x42, 0xaf, 0xc2, 0x44, - 0xc7, 0x8f, 0x38, 0x87, 0xe2, 0x6c, 0x88, 0x70, 0x8c, 0xa5, 0xc4, 0xba, 0xe1, 0xb6, 0x0e, 0xc4, - 0x26, 0x2e, 0x5a, 0x80, 0x91, 0xd8, 0x61, 0x36, 0x11, 0xc3, 0xf9, 0xc6, 0x9c, 0xeb, 0x14, 0x43, - 0x0f, 0xc8, 0x4c, 0x2b, 0x60, 0x51, 0x11, 0xbd, 0x25, 0x9d, 0x33, 0xf8, 0x59, 0x2f, 0xac, 0xa8, - 0x07, 0xbb, 0x17, 0x34, 0xd7, 0x0c, 0x61, 0x9d, 0x6d, 0xd0, 0xb2, 0xbf, 0x5e, 0x06, 0x48, 0xd8, - 0x71, 0xf4, 0x5e, 0xd7, 0x79, 0xf4, 0x5c, 0x6f, 0x06, 0xfe, 0xf8, 0x0e, 0x23, 0xf4, 0x7d, 0x16, - 0x8c, 0x39, 0xad, 0x56, 0xd0, 0x74, 0x62, 0x36, 0xca, 0x85, 0xde, 0xe7, 0xa1, 0x68, 0x7f, 0x21, - 0xa9, 0xc1, 0xbb, 0xf0, 0xa2, 0x5c, 0x78, 0x1a, 0xa4, 0x6f, 0x2f, 0xf4, 0x86, 0xd1, 0x27, 0xa4, - 0x94, 0xc6, 0x97, 0xc7, 0x5c, 0x5a, 0x4a, 0x2b, 0xb3, 0xa3, 0x5f, 0x13, 0xd0, 0xd0, 0x6d, 0x23, - 0xd2, 0xde, 0x50, 0x7e, 0xd0, 0x09, 0x83, 0x2b, 0xed, 0x17, 0x64, 0x0f, 0xd5, 0x75, 0x6f, 0xb2, - 0xe1, 0xfc, 0xc8, 0x2c, 0x9a, 0xf8, 0xd3, 0xc7, 0x93, 0xec, 0x1d, 0x98, 0x72, 0xcd, 0xbb, 0x5d, - 0xac, 0xa6, 0xa7, 0xf3, 0xe8, 0xa6, 0x58, 0x81, 0xe4, 0x36, 0x4f, 0x01, 0x70, 0x9a, 0x30, 0xaa, - 0x73, 0xbf, 0xbe, 0x9a, 0xbf, 0x19, 0x08, 0x6b, 0x7c, 0x3b, 0x77, 0x2e, 0xf7, 0xa3, 0x98, 0xec, - 0x52, 0xcc, 0xe4, 0xd2, 0x5e, 0x13, 0x75, 0xb1, 0xa2, 0x82, 0xde, 0x80, 0x11, 0xe6, 0x18, 0x1c, - 0xcd, 0x96, 0xf2, 0x95, 0x89, 0x66, 0x4c, 0x8b, 0x64, 0x53, 0xb1, 0xbf, 0x11, 0x16, 0x14, 0xd0, - 0x75, 0x19, 0xf8, 0x26, 0xaa, 0xf9, 0xb7, 0x23, 0xc2, 0x02, 0xdf, 0x94, 0x17, 0x3f, 0x9a, 0xc4, - 0xb4, 0xe1, 0xe5, 0x99, 0xa9, 0x17, 0x8c, 0x9a, 0x94, 0x39, 0x12, 0xff, 0x65, 0x46, 0x87, 0x59, - 0xc8, 0xef, 0x9e, 0x99, 0xf5, 0x21, 0x19, 0xce, 0x3b, 0x26, 0x09, 0x9c, 0xa6, 0x49, 0x19, 0x4d, - 0xbe, 0x73, 0x85, 0x3d, 0x7f, 0xbf, 0xfd, 0xcf, 0xe5, 0x6b, 0x76, 0xc9, 0xf0, 0x12, 0x2c, 0xea, - 0x9f, 0xe8, 0xad, 0x3f, 0xe7, 0xc3, 0x74, 0x7a, 0x8b, 0x3e, 0x52, 0x2e, 0xe3, 0xf7, 0x86, 0x60, - 0xd2, 0x5c, 0x52, 0xe8, 0x0a, 0x94, 0x05, 0x11, 0x15, 0x85, 0x55, 0xed, 0x92, 0x55, 0x09, 0xc0, - 0x09, 0x0e, 0x0b, 0xbe, 0xcb, 0xaa, 0x6b, 0x76, 0x98, 0x49, 0xf0, 0x5d, 0x05, 0xc1, 0x1a, 0x16, - 0x95, 0x97, 0x36, 0x82, 0x20, 0x56, 0x97, 0x8a, 0x5a, 0x77, 0x8b, 0xac, 0x14, 0x0b, 0x28, 0xbd, - 0x4c, 0x76, 0x48, 0xe8, 0x93, 0x96, 0x19, 0xdc, 0x4d, 0x5d, 0x26, 0x37, 0x74, 0x20, 0x36, 0x71, - 0xe9, 0x2d, 0x19, 0x44, 0x6c, 0x21, 0x0b, 0xa9, 0x2c, 0xb1, 0x6b, 0x6d, 0x70, 0x17, 0x7b, 0x09, - 0x47, 0x9f, 0x83, 0xc7, 0x95, 0x47, 0x3c, 0xe6, 0x8a, 0x6a, 0xd9, 0xe2, 0x88, 0xa1, 0x44, 0x79, - 0x7c, 0x29, 0x1b, 0x0d, 0xe7, 0xd5, 0x47, 0xaf, 0xc3, 0xa4, 0xe0, 0xdc, 0x25, 0xc5, 0x51, 0xd3, - 0x76, 0xe2, 0x86, 0x01, 0xc5, 0x29, 0x6c, 0x19, 0x9e, 0x8e, 0x31, 0xcf, 0x92, 0x42, 0xa9, 0x3b, - 0x3c, 0x9d, 0x0e, 0xc7, 0x5d, 0x35, 0xd0, 0x02, 0x4c, 0x71, 0xd6, 0xca, 0xf3, 0xb7, 0xf8, 0x9c, - 0x08, 0x77, 0x1b, 0xb5, 0xa5, 0x6e, 0x99, 0x60, 0x9c, 0xc6, 0x47, 0xaf, 0xc0, 0xb8, 0x13, 0x36, - 0xb7, 0xbd, 0x98, 0x34, 0xe3, 0x4e, 0xc8, 0xfd, 0x70, 0x34, 0xe3, 0x93, 0x05, 0x0d, 0x86, 0x0d, - 0x4c, 0xfb, 0x3d, 0x38, 0x95, 0xe1, 0xa9, 0x47, 0x17, 0x8e, 0xd3, 0xf6, 0xe4, 0x37, 0xa5, 0x2c, - 0x54, 0x17, 0xea, 0x35, 0xf9, 0x35, 0x1a, 0x16, 0x5d, 0x9d, 0xcc, 0xa3, 0x4f, 0x4b, 0xe0, 0xa2, - 0x56, 0xe7, 0x8a, 0x04, 0xe0, 0x04, 0xc7, 0xfe, 0xdf, 0x05, 0x98, 0xca, 0x50, 0xbe, 0xb3, 0x24, - 0x22, 0x29, 0xd9, 0x23, 0xc9, 0x19, 0x62, 0x46, 0x3b, 0x2c, 0x1c, 0x21, 0xda, 0x61, 0xb1, 0x5f, - 0xb4, 0xc3, 0xa1, 0xf7, 0x13, 0xed, 0xd0, 0x1c, 0xb1, 0xe1, 0x81, 0x46, 0x2c, 0x23, 0x42, 0xe2, - 0xc8, 0x11, 0x23, 0x24, 0x1a, 0x83, 0x3e, 0x3a, 0xc0, 0xa0, 0x7f, 0xb5, 0x00, 0xd3, 0x69, 0x23, - 0xb9, 0x13, 0x50, 0xc7, 0xbe, 0x61, 0xa8, 0x63, 0xb3, 0x53, 0xf2, 0xa4, 0x4d, 0xf7, 0xf2, 0x54, - 0xb3, 0x38, 0xa5, 0x9a, 0xfd, 0xf8, 0x40, 0xd4, 0x7a, 0xab, 0x69, 0xff, 0x41, 0x01, 0xce, 0xa4, - 0xab, 0x2c, 0xb5, 0x1c, 0x6f, 0xf7, 0x04, 0xc6, 0xe6, 0x96, 0x31, 0x36, 0xcf, 0x0f, 0xf2, 0x35, - 0xac, 0x6b, 0xb9, 0x03, 0x74, 0x37, 0x35, 0x40, 0x57, 0x06, 0x27, 0xd9, 0x7b, 0x94, 0xbe, 0x51, - 0x84, 0x0b, 0x99, 0xf5, 0x12, 0x6d, 0xe6, 0x8a, 0xa1, 0xcd, 0xbc, 0x9a, 0xd2, 0x66, 0xda, 0xbd, - 0x6b, 0x1f, 0x8f, 0x7a, 0x53, 0xb8, 0x50, 0xb2, 0x88, 0x78, 0x0f, 0xa9, 0xda, 0x34, 0x5c, 0x28, - 0x15, 0x21, 0x6c, 0xd2, 0xfd, 0x8b, 0xa4, 0xd2, 0xfc, 0xf7, 0x16, 0x9c, 0xcd, 0x9c, 0x9b, 0x13, - 0x50, 0x61, 0xad, 0x99, 0x2a, 0xac, 0x67, 0x06, 0x5e, 0xad, 0x39, 0x3a, 0xad, 0x5f, 0x1f, 0xca, - 0xf9, 0x16, 0x26, 0xa0, 0xdf, 0x82, 0x31, 0xa7, 0xd9, 0x24, 0x51, 0xb4, 0x1a, 0xb8, 0x2a, 0x42, - 0xdc, 0xf3, 0x4c, 0xce, 0x4a, 0x8a, 0x0f, 0x0f, 0x2a, 0x73, 0x69, 0x12, 0x09, 0x18, 0xeb, 0x14, - 0xcc, 0xa0, 0x96, 0x85, 0x63, 0x0d, 0x6a, 0x79, 0x15, 0x60, 0x4f, 0x71, 0xeb, 0x69, 0x21, 0x5f, - 0xe3, 0xe3, 0x35, 0x2c, 0xf4, 0x79, 0x28, 0x45, 0xe2, 0x1a, 0x17, 0x4b, 0xf1, 0xc5, 0x01, 0xe7, - 0xca, 0xd9, 0x20, 0x2d, 0xd3, 0x57, 0x5f, 0xe9, 0x43, 0x14, 0x49, 0xf4, 0x6d, 0x30, 0x1d, 0xf1, - 0x50, 0x30, 0x4b, 0x2d, 0x27, 0x62, 0x7e, 0x10, 0x62, 0x15, 0x32, 0x07, 0xfc, 0x46, 0x0a, 0x86, - 0xbb, 0xb0, 0xd1, 0x8a, 0xfc, 0x28, 0x16, 0xb7, 0x86, 0x2f, 0xcc, 0x4b, 0xc9, 0x07, 0x89, 0x14, - 0x66, 0xa7, 0xd3, 0xc3, 0xcf, 0x06, 0x5e, 0xab, 0x89, 0x3e, 0x0f, 0x40, 0x97, 0x8f, 0xd0, 0x25, - 0x8c, 0xe6, 0x1f, 0x9e, 0xf4, 0x54, 0x71, 0x33, 0x2d, 0x3f, 0x99, 0xf3, 0x62, 0x55, 0x11, 0xc1, - 0x1a, 0x41, 0xfb, 0xab, 0x43, 0xf0, 0x44, 0x8f, 0x33, 0x12, 0x2d, 0x98, 0x4f, 0xa0, 0xcf, 0xa6, - 0x85, 0xeb, 0xb9, 0xcc, 0xca, 0x86, 0xb4, 0x9d, 0x5a, 0x8a, 0x85, 0xf7, 0xbd, 0x14, 0x7f, 0xc0, - 0xd2, 0xd4, 0x1e, 0xdc, 0x98, 0xef, 0x33, 0x47, 0x3c, 0xfb, 0x8f, 0x51, 0x0f, 0xb2, 0x99, 0xa1, - 0x4c, 0xb8, 0x3a, 0x70, 0x77, 0x06, 0xd6, 0x2e, 0x9c, 0xac, 0xf2, 0xf7, 0x4b, 0x16, 0x3c, 0x99, - 0xd9, 0x5f, 0xc3, 0x64, 0xe3, 0x0a, 0x94, 0x9b, 0xb4, 0x50, 0xf3, 0x55, 0x4b, 0x9c, 0x78, 0x25, - 0x00, 0x27, 0x38, 0x86, 0x65, 0x46, 0xa1, 0xaf, 0x65, 0xc6, 0xbf, 0xb6, 0xa0, 0x6b, 0x7f, 0x9c, - 0xc0, 0x41, 0x5d, 0x33, 0x0f, 0xea, 0x8f, 0x0e, 0x32, 0x97, 0x39, 0x67, 0xf4, 0x1f, 0x4d, 0xc1, - 0x63, 0x39, 0xbe, 0x1a, 0x7b, 0x30, 0xb3, 0xd5, 0x24, 0xa6, 0x17, 0xa0, 0xf8, 0x98, 0x4c, 0x87, - 0xc9, 0x9e, 0x2e, 0x83, 0x2c, 0x1f, 0xd1, 0x4c, 0x17, 0x0a, 0xee, 0x6e, 0x02, 0x7d, 0xc9, 0x82, - 0xd3, 0xce, 0xbd, 0xa8, 0x2b, 0x81, 0xa9, 0x58, 0x33, 0x2f, 0x65, 0x2a, 0x41, 0xfa, 0x24, 0x3c, - 0xe5, 0x09, 0x9a, 0xb2, 0xb0, 0x70, 0x66, 0x5b, 0x08, 0x8b, 0x98, 0xa1, 0x94, 0x9d, 0xef, 0xe1, - 0xa7, 0x9a, 0xe5, 0x54, 0xc3, 0x8f, 0x6c, 0x09, 0xc1, 0x8a, 0x0e, 0xfa, 0x22, 0x94, 0xb7, 0xa4, - 0xa7, 0x5b, 0xc6, 0x95, 0x90, 0x0c, 0x64, 0x6f, 0xff, 0x3f, 0xfe, 0x40, 0xa9, 0x90, 0x70, 0x42, - 0x14, 0xbd, 0x0e, 0x45, 0x7f, 0x33, 0xea, 0x95, 0xe3, 0x28, 0x65, 0xd3, 0xc4, 0xbd, 0xc1, 0xd7, - 0x56, 0x1a, 0x98, 0x56, 0x44, 0xd7, 0xa1, 0x18, 0x6e, 0xb8, 0x42, 0x83, 0x97, 0x79, 0x86, 0xe3, - 0xc5, 0x6a, 0x4e, 0xaf, 0x18, 0x25, 0xbc, 0x58, 0xc5, 0x94, 0x04, 0xaa, 0xc3, 0x30, 0x73, 0x70, - 0x10, 0xf7, 0x41, 0x26, 0xe7, 0xdb, 0xc3, 0x51, 0x88, 0xbb, 0x8c, 0x33, 0x04, 0xcc, 0x09, 0xa1, - 0x75, 0x18, 0x69, 0xb2, 0x7c, 0x38, 0x22, 0x60, 0xf5, 0x27, 0x32, 0x75, 0x75, 0x3d, 0x12, 0x05, - 0x09, 0xd5, 0x15, 0xc3, 0xc0, 0x82, 0x16, 0xa3, 0x4a, 0xda, 0xdb, 0x9b, 0x91, 0xc8, 0xdf, 0x96, - 0x4d, 0xb5, 0x47, 0xfe, 0x2b, 0x41, 0x95, 0x61, 0x60, 0x41, 0x0b, 0x7d, 0x1a, 0x0a, 0x9b, 0x4d, - 0xe1, 0xff, 0x90, 0xa9, 0xb4, 0x33, 0x1d, 0xfa, 0x17, 0x47, 0x1e, 0x1c, 0x54, 0x0a, 0x2b, 0x4b, - 0xb8, 0xb0, 0xd9, 0x44, 0x6b, 0x30, 0xba, 0xc9, 0x5d, 0x80, 0x85, 0x5e, 0xee, 0xe9, 0x6c, 0xef, - 0xe4, 0x2e, 0x2f, 0x61, 0x6e, 0xb7, 0x2f, 0x00, 0x58, 0x12, 0x61, 0x21, 0x38, 0x95, 0x2b, 0xb3, - 0x88, 0x45, 0x3d, 0x7f, 0x34, 0xf7, 0x73, 0x7e, 0x3f, 0x27, 0x0e, 0xd1, 0x58, 0xa3, 0x48, 0x57, - 0xb5, 0x23, 0x93, 0x68, 0x8a, 0x58, 0x1d, 0x99, 0xab, 0xba, 0x4f, 0x7e, 0x51, 0xbe, 0xaa, 0x15, - 0x12, 0x4e, 0x88, 0xa2, 0x1d, 0x98, 0xd8, 0x8b, 0xda, 0xdb, 0x44, 0x6e, 0x69, 0x16, 0xba, 0x23, - 0xe7, 0x0a, 0xbb, 0x23, 0x10, 0xbd, 0x30, 0xee, 0x38, 0xad, 0xae, 0x53, 0x88, 0xbd, 0x6a, 0xdf, - 0xd1, 0x89, 0x61, 0x93, 0x36, 0x1d, 0xfe, 0x77, 0x3b, 0xc1, 0xc6, 0x7e, 0x4c, 0x44, 0xf0, 0xea, - 0xcc, 0xe1, 0x7f, 0x93, 0xa3, 0x74, 0x0f, 0xbf, 0x00, 0x60, 0x49, 0x04, 0xdd, 0x11, 0xc3, 0xc3, - 0x4e, 0xcf, 0xe9, 0xfc, 0x60, 0x4a, 0x99, 0x59, 0x6c, 0xb5, 0x41, 0x61, 0xa7, 0x65, 0x42, 0x8a, - 0x9d, 0x92, 0xed, 0xed, 0x20, 0x0e, 0xfc, 0xd4, 0x09, 0x3d, 0x93, 0x7f, 0x4a, 0xd6, 0x33, 0xf0, - 0xbb, 0x4f, 0xc9, 0x2c, 0x2c, 0x9c, 0xd9, 0x16, 0x72, 0x61, 0xb2, 0x1d, 0x84, 0xf1, 0xbd, 0x20, - 0x94, 0xeb, 0x0b, 0xf5, 0xd0, 0x2b, 0x18, 0x98, 0xa2, 0x45, 0x16, 0x4c, 0xdd, 0x84, 0xe0, 0x14, - 0x4d, 0xf4, 0x59, 0x18, 0x8d, 0x9a, 0x4e, 0x8b, 0xd4, 0x6e, 0xcd, 0x9e, 0xca, 0xbf, 0x7e, 0x1a, - 0x1c, 0x25, 0x67, 0x75, 0xb1, 0xc9, 0x11, 0x28, 0x58, 0x92, 0x43, 0x2b, 0x30, 0xcc, 0x32, 0x22, - 0xb0, 0xb8, 0xdb, 0x39, 0x31, 0xa1, 0xba, 0x2c, 0x4c, 0xf9, 0xd9, 0xc4, 0x8a, 0x31, 0xaf, 0x4e, - 0xf7, 0x80, 0x60, 0xaf, 0x83, 0x68, 0xf6, 0x4c, 0xfe, 0x1e, 0x10, 0x5c, 0xf9, 0xad, 0x46, 0xaf, - 0x3d, 0xa0, 0x90, 0x70, 0x42, 0x94, 0x9e, 0xcc, 0xf4, 0x34, 0x7d, 0xac, 0x87, 0x41, 0x4b, 0xee, - 0x59, 0xca, 0x4e, 0x66, 0x7a, 0x92, 0x52, 0x12, 0xf6, 0xef, 0x8c, 0x76, 0xf3, 0x2c, 0x4c, 0x20, - 0xfb, 0xab, 0x56, 0xd7, 0x5b, 0xdd, 0x27, 0x07, 0xd5, 0x0f, 0x1d, 0x23, 0xb7, 0xfa, 0x25, 0x0b, - 0x1e, 0x6b, 0x67, 0x7e, 0x88, 0x60, 0x00, 0x06, 0x53, 0x33, 0xf1, 0x4f, 0x57, 0xb1, 0xf1, 0xb3, - 0xe1, 0x38, 0xa7, 0xa5, 0xb4, 0x44, 0x50, 0x7c, 0xdf, 0x12, 0xc1, 0x2a, 0x94, 0x18, 0x93, 0xd9, - 0x27, 0x3f, 0x5c, 0x5a, 0x30, 0x62, 0xac, 0xc4, 0x92, 0xa8, 0x88, 0x15, 0x09, 0xf4, 0x83, 0x16, - 0x9c, 0x4f, 0x77, 0x1d, 0x13, 0x06, 0x16, 0x91, 0xe4, 0xb9, 0x2c, 0xb8, 0x22, 0xbe, 0xff, 0x7c, - 0xbd, 0x17, 0xf2, 0x61, 0x3f, 0x04, 0xdc, 0xbb, 0x31, 0x54, 0xcd, 0x10, 0x46, 0x47, 0x4c, 0x05, - 0xfc, 0x00, 0x02, 0xe9, 0x4b, 0x30, 0xbe, 0x1b, 0x74, 0xfc, 0x58, 0xd8, 0xbf, 0x08, 0x8f, 0x45, - 0xf6, 0xe0, 0xbc, 0xaa, 0x95, 0x63, 0x03, 0x2b, 0x25, 0xc6, 0x96, 0x1e, 0x5a, 0x8c, 0x7d, 0x3b, - 0x95, 0x50, 0xbe, 0x9c, 0x1f, 0xb1, 0x50, 0x48, 0xfc, 0x47, 0x48, 0x2b, 0x7f, 0xb2, 0xb2, 0xd1, - 0x4f, 0x59, 0x19, 0x4c, 0x3d, 0x97, 0x96, 0x5f, 0x33, 0xa5, 0xe5, 0x4b, 0x69, 0x69, 0xb9, 0x4b, - 0xf9, 0x6a, 0x08, 0xca, 0x83, 0x87, 0xbd, 0x1e, 0x34, 0x8e, 0x9c, 0xdd, 0x82, 0x8b, 0xfd, 0xae, - 0x25, 0x66, 0x08, 0xe5, 0xaa, 0xa7, 0xb6, 0xc4, 0x10, 0xca, 0xad, 0x55, 0x31, 0x83, 0x0c, 0x1a, - 0x68, 0xc4, 0xfe, 0x9f, 0x16, 0x14, 0xeb, 0x81, 0x7b, 0x02, 0xca, 0xe4, 0xcf, 0x18, 0xca, 0xe4, - 0x27, 0x72, 0x12, 0xfd, 0xe7, 0xaa, 0x8e, 0x97, 0x53, 0xaa, 0xe3, 0xf3, 0x79, 0x04, 0x7a, 0x2b, - 0x8a, 0x7f, 0xbc, 0x08, 0x63, 0xf5, 0xc0, 0x55, 0x56, 0xc8, 0xbf, 0xfe, 0x30, 0x56, 0xc8, 0xb9, - 0x61, 0x61, 0x35, 0xca, 0xcc, 0x7e, 0x4a, 0x3a, 0xe1, 0xfd, 0x39, 0x33, 0x46, 0xbe, 0x4b, 0xbc, - 0xad, 0xed, 0x98, 0xb8, 0xe9, 0xcf, 0x39, 0x39, 0x63, 0xe4, 0xff, 0x61, 0xc1, 0x54, 0xaa, 0x75, - 0xd4, 0x82, 0x89, 0x96, 0xae, 0x09, 0x14, 0xeb, 0xf4, 0xa1, 0x94, 0x88, 0xc2, 0x98, 0x53, 0x2b, - 0xc2, 0x26, 0x71, 0x34, 0x0f, 0xa0, 0x5e, 0xea, 0xa4, 0x06, 0x8c, 0x71, 0xfd, 0xea, 0x29, 0x2f, - 0xc2, 0x1a, 0x06, 0x7a, 0x19, 0xc6, 0xe2, 0xa0, 0x1d, 0xb4, 0x82, 0xad, 0xfd, 0x1b, 0x44, 0x86, - 0xb6, 0x51, 0x26, 0x5a, 0xeb, 0x09, 0x08, 0xeb, 0x78, 0xf6, 0x4f, 0x16, 0xf9, 0x87, 0xfa, 0xb1, - 0xf7, 0xcd, 0x35, 0xf9, 0xe1, 0x5e, 0x93, 0xdf, 0xb0, 0x60, 0x9a, 0xb6, 0xce, 0xcc, 0x45, 0xe4, - 0x65, 0xab, 0xd2, 0xef, 0x58, 0x3d, 0xd2, 0xef, 0x5c, 0xa2, 0x67, 0x97, 0x1b, 0x74, 0x62, 0xa1, - 0x41, 0xd3, 0x0e, 0x27, 0x5a, 0x8a, 0x05, 0x54, 0xe0, 0x91, 0x30, 0x14, 0x3e, 0x50, 0x3a, 0x1e, - 0x09, 0x43, 0x2c, 0xa0, 0x32, 0x3b, 0xcf, 0x50, 0x4e, 0x76, 0x1e, 0x16, 0xa8, 0x4f, 0x18, 0x16, - 0x08, 0xb6, 0x47, 0x0b, 0xd4, 0x27, 0x2d, 0x0e, 0x12, 0x1c, 0xfb, 0xe7, 0x8a, 0x30, 0x5e, 0x0f, - 0xdc, 0xe4, 0xad, 0xec, 0x25, 0xe3, 0xad, 0xec, 0x62, 0xea, 0xad, 0x6c, 0x5a, 0xc7, 0xfd, 0xe6, - 0xcb, 0xd8, 0x07, 0xf5, 0x32, 0xf6, 0xaf, 0x2c, 0x36, 0x6b, 0xd5, 0xb5, 0x86, 0xc8, 0x0e, 0xfc, - 0x02, 0x8c, 0xb1, 0x03, 0x89, 0x39, 0xdd, 0xc9, 0x07, 0x24, 0x16, 0x78, 0x7f, 0x2d, 0x29, 0xc6, - 0x3a, 0x0e, 0xba, 0x0c, 0xa5, 0x88, 0x38, 0x61, 0x73, 0x5b, 0x9d, 0x71, 0xe2, 0x79, 0x85, 0x97, - 0x61, 0x05, 0x45, 0x6f, 0x26, 0x31, 0xe2, 0x8a, 0xf9, 0x79, 0x6e, 0xf5, 0xfe, 0xf0, 0x2d, 0x92, - 0x1f, 0x18, 0xce, 0xbe, 0x0b, 0xa8, 0x1b, 0x7f, 0x80, 0xe0, 0x48, 0x15, 0x33, 0x38, 0x52, 0xb9, - 0x2b, 0x30, 0xd2, 0x9f, 0x5a, 0x30, 0x59, 0x0f, 0x5c, 0xba, 0x75, 0xff, 0x22, 0xed, 0x53, 0x3d, - 0x40, 0xe6, 0x48, 0x8f, 0x00, 0x99, 0xff, 0xd0, 0x82, 0xd1, 0x7a, 0xe0, 0x9e, 0x80, 0xde, 0xfd, - 0x35, 0x53, 0xef, 0xfe, 0x78, 0xce, 0x92, 0xc8, 0x51, 0xb5, 0xff, 0x42, 0x11, 0x26, 0x68, 0x3f, - 0x83, 0x2d, 0x39, 0x4b, 0xc6, 0x88, 0x58, 0x03, 0x8c, 0x08, 0x65, 0x73, 0x83, 0x56, 0x2b, 0xb8, - 0x97, 0x9e, 0xb1, 0x15, 0x56, 0x8a, 0x05, 0x14, 0x3d, 0x07, 0xa5, 0x76, 0x48, 0xf6, 0xbc, 0x40, - 0xf0, 0x8f, 0xda, 0x2b, 0x46, 0x5d, 0x94, 0x63, 0x85, 0x41, 0xe5, 0xae, 0xc8, 0xf3, 0x9b, 0x44, - 0x26, 0xd9, 0x1e, 0x62, 0x79, 0xb8, 0x78, 0xe4, 0x6b, 0xad, 0x1c, 0x1b, 0x58, 0xe8, 0x2e, 0x94, - 0xd9, 0x7f, 0x76, 0xa2, 0x1c, 0x3d, 0x6f, 0x90, 0x48, 0x37, 0x21, 0x08, 0xe0, 0x84, 0x16, 0xba, - 0x0a, 0x10, 0xcb, 0xe8, 0xc8, 0x91, 0x88, 0x71, 0xa3, 0x78, 0x6d, 0x15, 0x37, 0x39, 0xc2, 0x1a, - 0x16, 0x7a, 0x16, 0xca, 0xb1, 0xe3, 0xb5, 0x6e, 0x7a, 0x3e, 0x89, 0x98, 0xca, 0xb9, 0x28, 0xb3, - 0x49, 0x88, 0x42, 0x9c, 0xc0, 0x29, 0xaf, 0xc3, 0x1c, 0xc0, 0x79, 0xd6, 0xb1, 0x12, 0xc3, 0x66, - 0xbc, 0xce, 0x4d, 0x55, 0x8a, 0x35, 0x0c, 0xfb, 0x15, 0x38, 0x53, 0x0f, 0xdc, 0x7a, 0x10, 0xc6, - 0x2b, 0x41, 0x78, 0xcf, 0x09, 0x5d, 0x39, 0x7f, 0x15, 0x99, 0xd8, 0x80, 0x9e, 0x3d, 0xc3, 0x7c, - 0x67, 0x1a, 0x29, 0x0b, 0x5e, 0x64, 0xdc, 0xce, 0x11, 0x9d, 0x3a, 0x9a, 0xec, 0xde, 0x55, 0x09, - 0x06, 0xaf, 0x39, 0x31, 0x41, 0xb7, 0x58, 0x52, 0xb2, 0xe4, 0x0a, 0x12, 0xd5, 0x9f, 0xd1, 0x92, - 0x92, 0x25, 0xc0, 0xcc, 0x3b, 0xcb, 0xac, 0x6f, 0xff, 0xcc, 0x10, 0x3b, 0x8d, 0x52, 0xf9, 0xf6, - 0xd0, 0x17, 0x60, 0x32, 0x22, 0x37, 0x3d, 0xbf, 0x73, 0x5f, 0x0a, 0xe1, 0x3d, 0xdc, 0x72, 0x1a, - 0xcb, 0x3a, 0x26, 0x57, 0xe5, 0x99, 0x65, 0x38, 0x45, 0x8d, 0xce, 0x53, 0xd8, 0xf1, 0x17, 0xa2, - 0xdb, 0x11, 0x09, 0x45, 0xbe, 0x37, 0x36, 0x4f, 0x58, 0x16, 0xe2, 0x04, 0x4e, 0xd7, 0x25, 0xfb, - 0xb3, 0x16, 0xf8, 0x38, 0x08, 0x62, 0xb9, 0x92, 0x59, 0xc6, 0x20, 0xad, 0x1c, 0x1b, 0x58, 0x68, - 0x05, 0x50, 0xd4, 0x69, 0xb7, 0x5b, 0xec, 0x61, 0xdf, 0x69, 0x5d, 0x0b, 0x83, 0x4e, 0x9b, 0xbf, - 0x7a, 0x16, 0x79, 0x60, 0xc2, 0x46, 0x17, 0x14, 0x67, 0xd4, 0xa0, 0xa7, 0xcf, 0x66, 0xc4, 0x7e, - 0xb3, 0xd5, 0x5d, 0x14, 0xea, 0xf5, 0x06, 0x2b, 0xc2, 0x12, 0x46, 0x17, 0x13, 0x6b, 0x9e, 0x63, - 0x8e, 0x24, 0x8b, 0x09, 0xab, 0x52, 0xac, 0x61, 0xa0, 0x65, 0x18, 0x8d, 0xf6, 0xa3, 0x66, 0x2c, - 0x22, 0x32, 0xe5, 0x64, 0xee, 0x6c, 0x30, 0x14, 0x2d, 0x9b, 0x04, 0xaf, 0x82, 0x65, 0x5d, 0xb4, - 0x0b, 0x93, 0xf7, 0x3c, 0xdf, 0x0d, 0xee, 0x45, 0x72, 0xa2, 0x4a, 0xf9, 0xaa, 0xd1, 0xbb, 0x1c, - 0x33, 0x35, 0xd9, 0xc6, 0xbc, 0xdd, 0x35, 0x88, 0xe1, 0x14, 0x71, 0xfb, 0xbb, 0xd8, 0xdd, 0xcb, - 0x92, 0x91, 0xc5, 0x9d, 0x90, 0xa0, 0x5d, 0x98, 0x68, 0xb3, 0x15, 0x26, 0x42, 0x65, 0x8b, 0x65, - 0xf2, 0xd2, 0x80, 0x42, 0xf4, 0x3d, 0x7a, 0xae, 0x29, 0x25, 0x17, 0x93, 0x4e, 0xea, 0x3a, 0x39, - 0x6c, 0x52, 0xb7, 0xbf, 0x8a, 0xd8, 0x11, 0xdf, 0xe0, 0x92, 0xf1, 0xa8, 0xb0, 0x64, 0x16, 0x62, - 0xc0, 0x5c, 0xbe, 0x8a, 0x26, 0x19, 0x40, 0x61, 0x0d, 0x8d, 0x65, 0x5d, 0xf4, 0x26, 0x7b, 0x14, - 0xe7, 0xe7, 0x6a, 0xbf, 0x9c, 0xd0, 0x1c, 0xcb, 0x78, 0xff, 0x16, 0x15, 0xb1, 0x46, 0x04, 0xdd, - 0x84, 0x09, 0x91, 0xbb, 0x4a, 0xe8, 0xe0, 0x8a, 0x86, 0x8e, 0x65, 0x02, 0xeb, 0xc0, 0xc3, 0x74, - 0x01, 0x36, 0x2b, 0xa3, 0x2d, 0x38, 0xaf, 0x25, 0x72, 0xbc, 0x16, 0x3a, 0xec, 0xa1, 0xd4, 0x63, - 0x7b, 0x56, 0x3b, 0xa6, 0x9f, 0x7c, 0x70, 0x50, 0x39, 0xbf, 0xde, 0x0b, 0x11, 0xf7, 0xa6, 0x83, - 0x6e, 0xc1, 0x19, 0xee, 0x30, 0x58, 0x25, 0x8e, 0xdb, 0xf2, 0x7c, 0x75, 0x0f, 0xf0, 0x65, 0x7f, - 0xf6, 0xc1, 0x41, 0xe5, 0xcc, 0x42, 0x16, 0x02, 0xce, 0xae, 0x87, 0x5e, 0x83, 0xb2, 0xeb, 0x47, - 0x62, 0x0c, 0x46, 0x8c, 0x1c, 0xa5, 0xe5, 0xea, 0x5a, 0x43, 0x7d, 0x7f, 0xf2, 0x07, 0x27, 0x15, - 0xd0, 0x16, 0xd7, 0xc3, 0x29, 0xb1, 0x77, 0x34, 0x3f, 0x1f, 0xbd, 0x58, 0x12, 0x86, 0xcb, 0x10, - 0x57, 0x40, 0x2b, 0x93, 0x5b, 0xc3, 0x9b, 0xc8, 0x20, 0x8c, 0xde, 0x00, 0x44, 0xf9, 0x42, 0xaf, - 0x49, 0x16, 0x9a, 0x2c, 0x62, 0x39, 0x53, 0x5b, 0x96, 0x0c, 0x17, 0x0d, 0xd4, 0xe8, 0xc2, 0xc0, - 0x19, 0xb5, 0xd0, 0x75, 0x7a, 0x6e, 0xea, 0xa5, 0xc2, 0x74, 0x58, 0xca, 0x12, 0xb3, 0x55, 0xd2, - 0x0e, 0x49, 0xd3, 0x89, 0x89, 0x6b, 0x52, 0xc4, 0xa9, 0x7a, 0xf4, 0xea, 0x56, 0xc9, 0x8b, 0xc0, - 0x8c, 0xd2, 0xd1, 0x9d, 0xc0, 0x88, 0x8a, 0xe1, 0xdb, 0x41, 0x14, 0xaf, 0x91, 0xf8, 0x5e, 0x10, - 0xee, 0x88, 0xa0, 0x68, 0x49, 0x7c, 0xce, 0x04, 0x84, 0x75, 0x3c, 0xca, 0x76, 0xb3, 0x57, 0xe9, - 0x5a, 0x95, 0x3d, 0x08, 0x96, 0x92, 0x7d, 0x72, 0x9d, 0x17, 0x63, 0x09, 0x97, 0xa8, 0xb5, 0xfa, - 0x12, 0x7b, 0xdc, 0x4b, 0xa1, 0xd6, 0xea, 0x4b, 0x58, 0xc2, 0x11, 0xe9, 0xce, 0xff, 0x3a, 0x99, - 0xaf, 0x44, 0xed, 0xbe, 0x7d, 0x06, 0x4c, 0x01, 0xeb, 0xc3, 0xb4, 0xca, 0x3c, 0xcb, 0xa3, 0xc5, - 0x45, 0xb3, 0x53, 0x6c, 0x91, 0x0c, 0x1e, 0x6a, 0x4e, 0xa9, 0xa5, 0x6b, 0x29, 0x4a, 0xb8, 0x8b, - 0xb6, 0x11, 0x37, 0x65, 0xba, 0x6f, 0xf2, 0xa9, 0x2b, 0x50, 0x8e, 0x3a, 0x1b, 0x6e, 0xb0, 0xeb, - 0x78, 0x3e, 0x7b, 0x8b, 0xd3, 0x78, 0xba, 0x86, 0x04, 0xe0, 0x04, 0x07, 0xad, 0x40, 0xc9, 0x91, - 0x3a, 0x67, 0x94, 0x1f, 0x24, 0x41, 0x69, 0x9a, 0xb9, 0xdf, 0xb0, 0xd4, 0x32, 0xab, 0xba, 0xe8, - 0x55, 0x98, 0x10, 0x6e, 0x62, 0x3c, 0x74, 0x04, 0x7b, 0x2b, 0xd3, 0xfc, 0x00, 0x1a, 0x3a, 0x10, - 0x9b, 0xb8, 0xe8, 0xf3, 0x30, 0x49, 0xa9, 0x24, 0x07, 0xdb, 0xec, 0xe9, 0x41, 0x4e, 0x44, 0x2d, - 0xa9, 0x88, 0x5e, 0x19, 0xa7, 0x88, 0x21, 0x17, 0xce, 0x39, 0x9d, 0x38, 0x60, 0x7a, 0x7b, 0x73, - 0xfd, 0xaf, 0x07, 0x3b, 0xc4, 0x67, 0x4f, 0x66, 0xa5, 0xc5, 0x8b, 0x0f, 0x0e, 0x2a, 0xe7, 0x16, - 0x7a, 0xe0, 0xe1, 0x9e, 0x54, 0xd0, 0x6d, 0x18, 0x8b, 0x83, 0x16, 0xb3, 0xc8, 0xa7, 0x17, 0xe2, - 0x63, 0xf9, 0x71, 0x87, 0xd6, 0x15, 0x9a, 0xae, 0xb3, 0x52, 0x55, 0xb1, 0x4e, 0x07, 0xad, 0xf3, - 0x3d, 0xc6, 0x22, 0xb2, 0x92, 0x68, 0xf6, 0xf1, 0xfc, 0x81, 0x51, 0x81, 0x5b, 0xcd, 0x2d, 0x28, - 0x6a, 0x62, 0x9d, 0x0c, 0xba, 0x06, 0x33, 0xed, 0xd0, 0x0b, 0xd8, 0xc2, 0x56, 0x6f, 0x26, 0xb3, - 0x66, 0x1e, 0x89, 0x7a, 0x1a, 0x01, 0x77, 0xd7, 0xa1, 0x32, 0xad, 0x2c, 0x9c, 0x3d, 0xcb, 0x93, - 0x92, 0x71, 0x3e, 0x9f, 0x97, 0x61, 0x05, 0x45, 0xab, 0xec, 0x5c, 0xe6, 0xd2, 0xe7, 0xec, 0x5c, - 0x7e, 0x70, 0x09, 0x5d, 0x4a, 0xe5, 0xec, 0x99, 0xfa, 0x8b, 0x13, 0x0a, 0xf4, 0xde, 0x88, 0xb6, - 0x9d, 0x90, 0xd4, 0xc3, 0xa0, 0x49, 0x22, 0x2d, 0x08, 0xf4, 0x13, 0x3c, 0x70, 0x24, 0xbd, 0x37, - 0x1a, 0x59, 0x08, 0x38, 0xbb, 0x1e, 0x72, 0xb5, 0x5c, 0xdc, 0x94, 0xeb, 0x8d, 0x66, 0xcf, 0xf5, - 0xb0, 0x6f, 0x4a, 0xb1, 0xc8, 0xc9, 0x5a, 0x34, 0x8a, 0x23, 0x9c, 0xa2, 0x89, 0xbe, 0x0d, 0xa6, - 0x45, 0x9c, 0xa5, 0x64, 0xdc, 0xcf, 0x27, 0x86, 0x93, 0x38, 0x05, 0xc3, 0x5d, 0xd8, 0x3c, 0xf4, - 0xb5, 0xb3, 0xd1, 0x22, 0x62, 0x11, 0xde, 0xf4, 0xfc, 0x9d, 0x68, 0xf6, 0x02, 0xfb, 0x6a, 0x11, - 0xfa, 0x3a, 0x0d, 0xc5, 0x19, 0x35, 0xe6, 0xbe, 0x15, 0x66, 0xba, 0x6e, 0xae, 0x23, 0x85, 0x8b, - 0xff, 0x93, 0x61, 0x28, 0xab, 0x37, 0x00, 0x74, 0xc5, 0x7c, 0xda, 0x39, 0x9b, 0x7e, 0xda, 0x29, - 0x51, 0x51, 0x44, 0x7f, 0xcd, 0x59, 0x37, 0xec, 0x02, 0x0b, 0xf9, 0xc9, 0xd9, 0x74, 0x61, 0xa2, - 0xaf, 0x8f, 0xa1, 0xa6, 0xd2, 0x29, 0x0e, 0xfc, 0x46, 0x34, 0xd4, 0x53, 0x4b, 0x34, 0x60, 0x6e, - 0x64, 0xf4, 0x14, 0x95, 0xc7, 0xdc, 0x5a, 0x3d, 0x9d, 0x2c, 0xb4, 0x4e, 0x0b, 0x31, 0x87, 0x31, - 0xb9, 0x95, 0xb2, 0x59, 0x4c, 0x6e, 0x1d, 0x7d, 0x48, 0xb9, 0x55, 0x12, 0xc0, 0x09, 0x2d, 0xd4, - 0x82, 0x99, 0xa6, 0x99, 0xe7, 0x55, 0xf9, 0x15, 0x3e, 0xd5, 0x37, 0xe3, 0x6a, 0x47, 0x4b, 0xaa, - 0xb7, 0x94, 0xa6, 0x82, 0xbb, 0x09, 0xa3, 0x57, 0xa1, 0xf4, 0x6e, 0x10, 0xb1, 0x45, 0x29, 0x78, - 0x0d, 0xe9, 0x7f, 0x55, 0x7a, 0xf3, 0x56, 0x83, 0x95, 0x1f, 0x1e, 0x54, 0xc6, 0xea, 0x81, 0x2b, - 0xff, 0x62, 0x55, 0x01, 0xdd, 0x87, 0x33, 0xc6, 0x09, 0xad, 0xba, 0x0b, 0x83, 0x77, 0xf7, 0xbc, - 0x68, 0xee, 0x4c, 0x2d, 0x8b, 0x12, 0xce, 0x6e, 0x80, 0x1e, 0x7b, 0x7e, 0x20, 0x72, 0x24, 0x4b, - 0x7e, 0x86, 0xb1, 0x2d, 0x65, 0xdd, 0xfb, 0x3e, 0x85, 0x80, 0xbb, 0xeb, 0xd8, 0xbf, 0xcc, 0x9f, - 0x4c, 0x84, 0x62, 0x95, 0x44, 0x9d, 0xd6, 0x49, 0xa4, 0xe0, 0x5a, 0x36, 0x74, 0xbe, 0x0f, 0xfd, - 0x2c, 0xf7, 0x6b, 0x16, 0x7b, 0x96, 0x5b, 0x27, 0xbb, 0xed, 0x16, 0x15, 0xef, 0x1f, 0x7d, 0xc7, - 0xdf, 0x84, 0x52, 0x2c, 0x5a, 0xeb, 0x95, 0x35, 0x4c, 0xeb, 0x14, 0x7b, 0x9a, 0x54, 0x9c, 0x8e, - 0x2c, 0xc5, 0x8a, 0x8c, 0xfd, 0xcf, 0xf9, 0x0c, 0x48, 0xc8, 0x09, 0xe8, 0xdf, 0xaa, 0xa6, 0xfe, - 0xad, 0xd2, 0xe7, 0x0b, 0x72, 0xf4, 0x70, 0xff, 0xcc, 0xec, 0x37, 0x13, 0x2a, 0x3f, 0xec, 0xef, + // 13056 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x70, 0x64, 0x57, + 0x56, 0xd8, 0xbe, 0x6e, 0x7d, 0xf5, 0xd1, 0xf7, 0x9d, 0x0f, 0x6b, 0xe4, 0x99, 0xe9, 0xf1, 0xf3, + 0xee, 0x78, 0xbc, 0xb6, 0x35, 0xeb, 0xb1, 0xbd, 0x36, 0x6b, 0xaf, 0x41, 0x52, 0x4b, 0x33, 0xf2, + 0x8c, 0x34, 0xed, 0xdb, 0x9a, 0x99, 0x5d, 0xe3, 0x5d, 0xfc, 0xd4, 0xef, 0xaa, 0xf5, 0xac, 0xd6, + 0x7b, 0xed, 0xf7, 0x5e, 0x6b, 0x46, 0x0e, 0x54, 0xc8, 0x12, 0x08, 0x5b, 0x90, 0xd4, 0x56, 0xb2, + 0x95, 0x0f, 0xa0, 0x48, 0x15, 0x21, 0x05, 0x04, 0x92, 0x0a, 0x81, 0x00, 0x61, 0x49, 0x42, 0x20, + 0xa9, 0x22, 0xf9, 0xb1, 0x21, 0xa9, 0x4a, 0x2d, 0x55, 0x54, 0x14, 0x10, 0xa9, 0x50, 0xfc, 0x08, + 0xa4, 0x42, 0xfe, 0xa0, 0x50, 0x21, 0x75, 0x3f, 0xdf, 0xbd, 0xaf, 0xdf, 0xeb, 0x6e, 0x8d, 0x35, + 0xb2, 0xa1, 0xf6, 0x5f, 0xf7, 0x3d, 0xe7, 0x9e, 0x7b, 0xdf, 0xfd, 0x3c, 0xe7, 0xdc, 0xf3, 0x01, + 0xaf, 0x6e, 0xbf, 0x12, 0xcd, 0x79, 0xc1, 0xd5, 0xed, 0xf6, 0x06, 0x09, 0x7d, 0x12, 0x93, 0xe8, + 0xea, 0x2e, 0xf1, 0xdd, 0x20, 0xbc, 0x2a, 0x00, 0x4e, 0xcb, 0xbb, 0x5a, 0x0f, 0x42, 0x72, 0x75, + 0xf7, 0xf9, 0xab, 0x0d, 0xe2, 0x93, 0xd0, 0x89, 0x89, 0x3b, 0xd7, 0x0a, 0x83, 0x38, 0x40, 0x88, + 0xe3, 0xcc, 0x39, 0x2d, 0x6f, 0x8e, 0xe2, 0xcc, 0xed, 0x3e, 0x3f, 0xfb, 0x5c, 0xc3, 0x8b, 0xb7, + 0xda, 0x1b, 0x73, 0xf5, 0x60, 0xe7, 0x6a, 0x23, 0x68, 0x04, 0x57, 0x19, 0xea, 0x46, 0x7b, 0x93, + 0xfd, 0x63, 0x7f, 0xd8, 0x2f, 0x4e, 0x62, 0xf6, 0xc5, 0xa4, 0x99, 0x1d, 0xa7, 0xbe, 0xe5, 0xf9, + 0x24, 0xdc, 0xbb, 0xda, 0xda, 0x6e, 0xb0, 0x76, 0x43, 0x12, 0x05, 0xed, 0xb0, 0x4e, 0xd2, 0x0d, + 0x77, 0xad, 0x15, 0x5d, 0xdd, 0x21, 0xb1, 0x93, 0xd1, 0xdd, 0xd9, 0xab, 0x79, 0xb5, 0xc2, 0xb6, + 0x1f, 0x7b, 0x3b, 0x9d, 0xcd, 0x7c, 0xba, 0x57, 0x85, 0xa8, 0xbe, 0x45, 0x76, 0x9c, 0x8e, 0x7a, + 0x2f, 0xe4, 0xd5, 0x6b, 0xc7, 0x5e, 0xf3, 0xaa, 0xe7, 0xc7, 0x51, 0x1c, 0xa6, 0x2b, 0xd9, 0xdf, + 0xb0, 0xe0, 0xd2, 0xfc, 0xbd, 0xda, 0x52, 0xd3, 0x89, 0x62, 0xaf, 0xbe, 0xd0, 0x0c, 0xea, 0xdb, + 0xb5, 0x38, 0x08, 0xc9, 0xdd, 0xa0, 0xd9, 0xde, 0x21, 0x35, 0x36, 0x10, 0xe8, 0x59, 0x18, 0xd9, + 0x65, 0xff, 0x57, 0x2a, 0x33, 0xd6, 0x25, 0xeb, 0x4a, 0x69, 0x61, 0xea, 0x37, 0xf6, 0xcb, 0x1f, + 0x3b, 0xd8, 0x2f, 0x8f, 0xdc, 0x15, 0xe5, 0x58, 0x61, 0xa0, 0xcb, 0x30, 0xb4, 0x19, 0xad, 0xef, + 0xb5, 0xc8, 0x4c, 0x81, 0xe1, 0x4e, 0x08, 0xdc, 0xa1, 0xe5, 0x1a, 0x2d, 0xc5, 0x02, 0x8a, 0xae, + 0x42, 0xa9, 0xe5, 0x84, 0xb1, 0x17, 0x7b, 0x81, 0x3f, 0x53, 0xbc, 0x64, 0x5d, 0x19, 0x5c, 0x98, + 0x16, 0xa8, 0xa5, 0xaa, 0x04, 0xe0, 0x04, 0x87, 0x76, 0x23, 0x24, 0x8e, 0x7b, 0xdb, 0x6f, 0xee, + 0xcd, 0x0c, 0x5c, 0xb2, 0xae, 0x8c, 0x24, 0xdd, 0xc0, 0xa2, 0x1c, 0x2b, 0x0c, 0xfb, 0x87, 0x0a, + 0x30, 0x32, 0xbf, 0xb9, 0xe9, 0xf9, 0x5e, 0xbc, 0x87, 0xee, 0xc2, 0x98, 0x1f, 0xb8, 0x44, 0xfe, + 0x67, 0x5f, 0x31, 0x7a, 0xed, 0xd2, 0x5c, 0xe7, 0x52, 0x9a, 0x5b, 0xd3, 0xf0, 0x16, 0xa6, 0x0e, + 0xf6, 0xcb, 0x63, 0x7a, 0x09, 0x36, 0xe8, 0x20, 0x0c, 0xa3, 0xad, 0xc0, 0x55, 0x64, 0x0b, 0x8c, + 0x6c, 0x39, 0x8b, 0x6c, 0x35, 0x41, 0x5b, 0x98, 0x3c, 0xd8, 0x2f, 0x8f, 0x6a, 0x05, 0x58, 0x27, + 0x82, 0x36, 0x60, 0x92, 0xfe, 0xf5, 0x63, 0x4f, 0xd1, 0x2d, 0x32, 0xba, 0x4f, 0xe6, 0xd1, 0xd5, + 0x50, 0x17, 0x4e, 0x1d, 0xec, 0x97, 0x27, 0x53, 0x85, 0x38, 0x4d, 0xd0, 0x7e, 0x1f, 0x26, 0xe6, + 0xe3, 0xd8, 0xa9, 0x6f, 0x11, 0x97, 0xcf, 0x20, 0x7a, 0x11, 0x06, 0x7c, 0x67, 0x87, 0x88, 0xf9, + 0xbd, 0x24, 0x06, 0x76, 0x60, 0xcd, 0xd9, 0x21, 0x87, 0xfb, 0xe5, 0xa9, 0x3b, 0xbe, 0xf7, 0x5e, + 0x5b, 0xac, 0x0a, 0x5a, 0x86, 0x19, 0x36, 0xba, 0x06, 0xe0, 0x92, 0x5d, 0xaf, 0x4e, 0xaa, 0x4e, + 0xbc, 0x25, 0xe6, 0x1b, 0x89, 0xba, 0x50, 0x51, 0x10, 0xac, 0x61, 0xd9, 0x0f, 0xa0, 0x34, 0xbf, + 0x1b, 0x78, 0x6e, 0x35, 0x70, 0x23, 0xb4, 0x0d, 0x93, 0xad, 0x90, 0x6c, 0x92, 0x50, 0x15, 0xcd, + 0x58, 0x97, 0x8a, 0x57, 0x46, 0xaf, 0x5d, 0xc9, 0xfc, 0x58, 0x13, 0x75, 0xc9, 0x8f, 0xc3, 0xbd, + 0x85, 0xc7, 0x44, 0x7b, 0x93, 0x29, 0x28, 0x4e, 0x53, 0xb6, 0xff, 0x6d, 0x01, 0xce, 0xcc, 0xbf, + 0xdf, 0x0e, 0x49, 0xc5, 0x8b, 0xb6, 0xd3, 0x2b, 0xdc, 0xf5, 0xa2, 0xed, 0xb5, 0x64, 0x04, 0xd4, + 0xd2, 0xaa, 0x88, 0x72, 0xac, 0x30, 0xd0, 0x73, 0x30, 0x4c, 0x7f, 0xdf, 0xc1, 0x2b, 0xe2, 0x93, + 0x4f, 0x09, 0xe4, 0xd1, 0x8a, 0x13, 0x3b, 0x15, 0x0e, 0xc2, 0x12, 0x07, 0xad, 0xc2, 0x68, 0x9d, + 0x6d, 0xc8, 0xc6, 0x6a, 0xe0, 0x12, 0x36, 0x99, 0xa5, 0x85, 0x67, 0x28, 0xfa, 0x62, 0x52, 0x7c, + 0xb8, 0x5f, 0x9e, 0xe1, 0x7d, 0x13, 0x24, 0x34, 0x18, 0xd6, 0xeb, 0x23, 0x5b, 0xed, 0xaf, 0x01, + 0x46, 0x09, 0x32, 0xf6, 0xd6, 0x15, 0x6d, 0xab, 0x0c, 0xb2, 0xad, 0x32, 0x96, 0xbd, 0x4d, 0xd0, + 0xf3, 0x30, 0xb0, 0xed, 0xf9, 0xee, 0xcc, 0x10, 0xa3, 0x75, 0x81, 0xce, 0xf9, 0x4d, 0xcf, 0x77, + 0x0f, 0xf7, 0xcb, 0xd3, 0x46, 0x77, 0x68, 0x21, 0x66, 0xa8, 0xf6, 0x1f, 0x5b, 0x50, 0x66, 0xb0, + 0x65, 0xaf, 0x49, 0xaa, 0x24, 0x8c, 0xbc, 0x28, 0x26, 0x7e, 0x6c, 0x0c, 0xe8, 0x35, 0x80, 0x88, + 0xd4, 0x43, 0x12, 0x6b, 0x43, 0xaa, 0x16, 0x46, 0x4d, 0x41, 0xb0, 0x86, 0x45, 0x0f, 0x84, 0x68, + 0xcb, 0x09, 0xd9, 0xfa, 0x12, 0x03, 0xab, 0x0e, 0x84, 0x9a, 0x04, 0xe0, 0x04, 0xc7, 0x38, 0x10, + 0x8a, 0xbd, 0x0e, 0x04, 0xf4, 0x59, 0x98, 0x4c, 0x1a, 0x8b, 0x5a, 0x4e, 0x5d, 0x0e, 0x20, 0xdb, + 0x32, 0x35, 0x13, 0x84, 0xd3, 0xb8, 0xf6, 0x3f, 0xb2, 0xc4, 0xe2, 0xa1, 0x5f, 0xfd, 0x11, 0xff, + 0x56, 0xfb, 0x97, 0x2c, 0x18, 0x5e, 0xf0, 0x7c, 0xd7, 0xf3, 0x1b, 0xe8, 0x1d, 0x18, 0xa1, 0x77, + 0x93, 0xeb, 0xc4, 0x8e, 0x38, 0xf7, 0x3e, 0xa5, 0xed, 0x2d, 0x75, 0x55, 0xcc, 0xb5, 0xb6, 0x1b, + 0xb4, 0x20, 0x9a, 0xa3, 0xd8, 0x74, 0xb7, 0xdd, 0xde, 0x78, 0x97, 0xd4, 0xe3, 0x55, 0x12, 0x3b, + 0xc9, 0xe7, 0x24, 0x65, 0x58, 0x51, 0x45, 0x37, 0x61, 0x28, 0x76, 0xc2, 0x06, 0x89, 0xc5, 0x01, + 0x98, 0x79, 0x50, 0xf1, 0x9a, 0x98, 0xee, 0x48, 0xe2, 0xd7, 0x49, 0x72, 0x2d, 0xac, 0xb3, 0xaa, + 0x58, 0x90, 0xb0, 0xff, 0xfa, 0x30, 0x9c, 0x5b, 0xac, 0xad, 0xe4, 0xac, 0xab, 0xcb, 0x30, 0xe4, + 0x86, 0xde, 0x2e, 0x09, 0xc5, 0x38, 0x2b, 0x2a, 0x15, 0x56, 0x8a, 0x05, 0x14, 0xbd, 0x02, 0x63, + 0xfc, 0x42, 0xba, 0xe1, 0xf8, 0x6e, 0x53, 0x0e, 0xf1, 0x69, 0x81, 0x3d, 0x76, 0x57, 0x83, 0x61, + 0x03, 0xf3, 0x88, 0x8b, 0xea, 0x72, 0x6a, 0x33, 0xe6, 0x5d, 0x76, 0x5f, 0xb6, 0x60, 0x8a, 0x37, + 0x33, 0x1f, 0xc7, 0xa1, 0xb7, 0xd1, 0x8e, 0x49, 0x34, 0x33, 0xc8, 0x4e, 0xba, 0xc5, 0xac, 0xd1, + 0xca, 0x1d, 0x81, 0xb9, 0xbb, 0x29, 0x2a, 0xfc, 0x10, 0x9c, 0x11, 0xed, 0x4e, 0xa5, 0xc1, 0xb8, + 0xa3, 0x59, 0xf4, 0x3d, 0x16, 0xcc, 0xd6, 0x03, 0x3f, 0x0e, 0x83, 0x66, 0x93, 0x84, 0xd5, 0xf6, + 0x46, 0xd3, 0x8b, 0xb6, 0xf8, 0x3a, 0xc5, 0x64, 0x93, 0x9d, 0x04, 0x39, 0x73, 0xa8, 0x90, 0xc4, + 0x1c, 0x5e, 0x3c, 0xd8, 0x2f, 0xcf, 0x2e, 0xe6, 0x92, 0xc2, 0x5d, 0x9a, 0x41, 0xdb, 0x80, 0xe8, + 0x55, 0x5a, 0x8b, 0x9d, 0x06, 0x49, 0x1a, 0x1f, 0xee, 0xbf, 0xf1, 0xb3, 0x07, 0xfb, 0x65, 0xb4, + 0xd6, 0x41, 0x02, 0x67, 0x90, 0x45, 0xef, 0xc1, 0x69, 0x5a, 0xda, 0xf1, 0xad, 0x23, 0xfd, 0x37, + 0x37, 0x73, 0xb0, 0x5f, 0x3e, 0xbd, 0x96, 0x41, 0x04, 0x67, 0x92, 0x46, 0xdf, 0x6d, 0xc1, 0xb9, + 0xe4, 0xf3, 0x97, 0x1e, 0xb4, 0x1c, 0xdf, 0x4d, 0x1a, 0x2e, 0xf5, 0xdf, 0x30, 0x3d, 0x93, 0xcf, + 0x2d, 0xe6, 0x51, 0xc2, 0xf9, 0x8d, 0xcc, 0x2e, 0xc2, 0x99, 0xcc, 0xd5, 0x82, 0xa6, 0xa0, 0xb8, + 0x4d, 0x38, 0x17, 0x54, 0xc2, 0xf4, 0x27, 0x3a, 0x0d, 0x83, 0xbb, 0x4e, 0xb3, 0x2d, 0x36, 0x0a, + 0xe6, 0x7f, 0x3e, 0x53, 0x78, 0xc5, 0xb2, 0xff, 0x5d, 0x11, 0x26, 0x17, 0x6b, 0x2b, 0x0f, 0xb5, + 0x0b, 0xf5, 0x6b, 0xa8, 0xd0, 0xf5, 0x1a, 0x4a, 0x2e, 0xb5, 0x62, 0xee, 0xa5, 0xf6, 0x97, 0x33, + 0xb6, 0xd0, 0x00, 0xdb, 0x42, 0xdf, 0x92, 0xb3, 0x85, 0x8e, 0x79, 0xe3, 0xec, 0xe6, 0xac, 0xa2, + 0x41, 0x36, 0x99, 0x99, 0x1c, 0xcb, 0xad, 0xa0, 0xee, 0x34, 0xd3, 0x47, 0xdf, 0x11, 0x97, 0xd2, + 0xf1, 0xcc, 0x63, 0x1d, 0xc6, 0x16, 0x9d, 0x96, 0xb3, 0xe1, 0x35, 0xbd, 0xd8, 0x23, 0x11, 0x7a, + 0x0a, 0x8a, 0x8e, 0xeb, 0x32, 0x6e, 0xab, 0xb4, 0x70, 0xe6, 0x60, 0xbf, 0x5c, 0x9c, 0x77, 0xe9, + 0xb5, 0x0f, 0x0a, 0x6b, 0x0f, 0x53, 0x0c, 0xf4, 0x49, 0x18, 0x70, 0xc3, 0xa0, 0x35, 0x53, 0x60, + 0x98, 0x74, 0xd7, 0x0d, 0x54, 0xc2, 0xa0, 0x95, 0x42, 0x65, 0x38, 0xf6, 0xaf, 0x16, 0xe0, 0xfc, + 0x22, 0x69, 0x6d, 0x2d, 0xd7, 0x72, 0xce, 0xef, 0x2b, 0x30, 0xb2, 0x13, 0xf8, 0x5e, 0x1c, 0x84, + 0x91, 0x68, 0x9a, 0xad, 0x88, 0x55, 0x51, 0x86, 0x15, 0x14, 0x5d, 0x82, 0x81, 0x56, 0xc2, 0x54, + 0x8e, 0x49, 0x86, 0x94, 0xb1, 0x93, 0x0c, 0x42, 0x31, 0xda, 0x11, 0x09, 0xc5, 0x8a, 0x51, 0x18, + 0x77, 0x22, 0x12, 0x62, 0x06, 0x49, 0x6e, 0x66, 0x7a, 0x67, 0x8b, 0x13, 0x3a, 0x75, 0x33, 0x53, + 0x08, 0xd6, 0xb0, 0x50, 0x15, 0x4a, 0x51, 0x6a, 0x66, 0xfb, 0xda, 0xa6, 0xe3, 0xec, 0xea, 0x56, + 0x33, 0x99, 0x10, 0x31, 0x6e, 0x94, 0xa1, 0x9e, 0x57, 0xf7, 0xd7, 0x0a, 0x80, 0xf8, 0x10, 0xfe, + 0x39, 0x1b, 0xb8, 0x3b, 0x9d, 0x03, 0xd7, 0xff, 0x96, 0x38, 0xae, 0xd1, 0xfb, 0x3f, 0x16, 0x9c, + 0x5f, 0xf4, 0x7c, 0x97, 0x84, 0x39, 0x0b, 0xf0, 0xd1, 0xc8, 0xb2, 0x47, 0x63, 0x1a, 0x8c, 0x25, + 0x36, 0x70, 0x0c, 0x4b, 0xcc, 0xfe, 0x23, 0x0b, 0x10, 0xff, 0xec, 0x8f, 0xdc, 0xc7, 0xde, 0xe9, + 0xfc, 0xd8, 0x63, 0x58, 0x16, 0xf6, 0x2d, 0x98, 0x58, 0x6c, 0x7a, 0xc4, 0x8f, 0x57, 0xaa, 0x8b, + 0x81, 0xbf, 0xe9, 0x35, 0xd0, 0x67, 0x60, 0x22, 0xf6, 0x76, 0x48, 0xd0, 0x8e, 0x6b, 0xa4, 0x1e, + 0xf8, 0x4c, 0x92, 0xb4, 0xae, 0x0c, 0x2e, 0xa0, 0x83, 0xfd, 0xf2, 0xc4, 0xba, 0x01, 0xc1, 0x29, + 0x4c, 0xfb, 0xb7, 0xe9, 0xf8, 0x05, 0x3b, 0xad, 0xc0, 0x27, 0x7e, 0xbc, 0x18, 0xf8, 0x2e, 0xd7, + 0x38, 0x7c, 0x06, 0x06, 0x62, 0x3a, 0x1e, 0x7c, 0xec, 0x2e, 0xcb, 0x8d, 0x42, 0x47, 0xe1, 0x70, + 0xbf, 0x7c, 0xb6, 0xb3, 0x06, 0x1b, 0x27, 0x56, 0x07, 0x7d, 0x0b, 0x0c, 0x45, 0xb1, 0x13, 0xb7, + 0x23, 0x31, 0x9a, 0x4f, 0xc8, 0xd1, 0xac, 0xb1, 0xd2, 0xc3, 0xfd, 0xf2, 0xa4, 0xaa, 0xc6, 0x8b, + 0xb0, 0xa8, 0x80, 0x9e, 0x86, 0xe1, 0x1d, 0x12, 0x45, 0x4e, 0x43, 0xde, 0x86, 0x93, 0xa2, 0xee, + 0xf0, 0x2a, 0x2f, 0xc6, 0x12, 0x8e, 0x9e, 0x84, 0x41, 0x12, 0x86, 0x41, 0x28, 0xf6, 0xe8, 0xb8, + 0x40, 0x1c, 0x5c, 0xa2, 0x85, 0x98, 0xc3, 0xec, 0xff, 0x68, 0xc1, 0xa4, 0xea, 0x2b, 0x6f, 0xeb, + 0x04, 0xa4, 0x82, 0xb7, 0x00, 0xea, 0xf2, 0x03, 0x23, 0x76, 0x7b, 0x8c, 0x5e, 0xbb, 0x9c, 0x79, + 0x51, 0x77, 0x0c, 0x63, 0x42, 0x59, 0x15, 0x45, 0x58, 0xa3, 0x66, 0xff, 0x4b, 0x0b, 0x4e, 0xa5, + 0xbe, 0xe8, 0x96, 0x17, 0xc5, 0xe8, 0xed, 0x8e, 0xaf, 0x9a, 0xeb, 0xef, 0xab, 0x68, 0x6d, 0xf6, + 0x4d, 0x6a, 0x29, 0xcb, 0x12, 0xed, 0x8b, 0x6e, 0xc0, 0xa0, 0x17, 0x93, 0x1d, 0xf9, 0x31, 0x4f, + 0x76, 0xfd, 0x18, 0xde, 0xab, 0x64, 0x46, 0x56, 0x68, 0x4d, 0xcc, 0x09, 0xd8, 0x7f, 0xab, 0x08, + 0x25, 0xbe, 0x6c, 0x57, 0x9d, 0xd6, 0x09, 0xcc, 0xc5, 0x0a, 0x0c, 0x30, 0xea, 0xbc, 0xe3, 0x4f, + 0x65, 0x77, 0x5c, 0x74, 0x67, 0x8e, 0x8a, 0xfc, 0x9c, 0x39, 0x52, 0x57, 0x03, 0x2d, 0xc2, 0x8c, + 0x04, 0x72, 0x00, 0x36, 0x3c, 0xdf, 0x09, 0xf7, 0x68, 0xd9, 0x4c, 0x91, 0x11, 0x7c, 0xae, 0x3b, + 0xc1, 0x05, 0x85, 0xcf, 0xc9, 0xaa, 0xbe, 0x26, 0x00, 0xac, 0x11, 0x9d, 0x7d, 0x19, 0x4a, 0x0a, + 0xf9, 0x28, 0x3c, 0xce, 0xec, 0x67, 0x61, 0x32, 0xd5, 0x56, 0xaf, 0xea, 0x63, 0x3a, 0x8b, 0xf4, + 0xcb, 0xec, 0x14, 0x10, 0xbd, 0x5e, 0xf2, 0x77, 0xc5, 0x29, 0xfa, 0x3e, 0x9c, 0x6e, 0x66, 0x1c, + 0x4e, 0x62, 0xaa, 0xfa, 0x3f, 0xcc, 0xce, 0x8b, 0xcf, 0x3e, 0x9d, 0x05, 0xc5, 0x99, 0x6d, 0xd0, + 0x6b, 0x3f, 0x68, 0xd1, 0x35, 0xef, 0x34, 0x75, 0x0e, 0xfa, 0xb6, 0x28, 0xc3, 0x0a, 0x4a, 0x8f, + 0xb0, 0xd3, 0xaa, 0xf3, 0x37, 0xc9, 0x5e, 0x8d, 0x34, 0x49, 0x3d, 0x0e, 0xc2, 0x0f, 0xb5, 0xfb, + 0x17, 0xf8, 0xe8, 0xf3, 0x13, 0x70, 0x54, 0x10, 0x28, 0xde, 0x24, 0x7b, 0x7c, 0x2a, 0xf4, 0xaf, + 0x2b, 0x76, 0xfd, 0xba, 0x9f, 0xb5, 0x60, 0x5c, 0x7d, 0xdd, 0x09, 0x6c, 0xf5, 0x05, 0x73, 0xab, + 0x5f, 0xe8, 0xba, 0xc0, 0x73, 0x36, 0xf9, 0xd7, 0x0a, 0x70, 0x4e, 0xe1, 0x50, 0x76, 0x9f, 0xff, + 0x11, 0xab, 0xea, 0x2a, 0x94, 0x7c, 0xa5, 0x88, 0xb2, 0x4c, 0x0d, 0x50, 0xa2, 0x86, 0x4a, 0x70, + 0x28, 0xd7, 0xe6, 0x27, 0xda, 0xa2, 0x31, 0x5d, 0x43, 0x2b, 0xb4, 0xb1, 0x0b, 0x50, 0x6c, 0x7b, + 0xae, 0xb8, 0x33, 0x3e, 0x25, 0x47, 0xfb, 0xce, 0x4a, 0xe5, 0x70, 0xbf, 0xfc, 0x44, 0xde, 0xeb, + 0x00, 0xbd, 0xac, 0xa2, 0xb9, 0x3b, 0x2b, 0x15, 0x4c, 0x2b, 0xa3, 0x79, 0x98, 0x94, 0x0f, 0x20, + 0x77, 0x29, 0x07, 0x15, 0xf8, 0xe2, 0x6a, 0x51, 0x6a, 0x56, 0x6c, 0x82, 0x71, 0x1a, 0x1f, 0x55, + 0x60, 0x6a, 0xbb, 0xbd, 0x41, 0x9a, 0x24, 0xe6, 0x1f, 0x7c, 0x93, 0x70, 0x25, 0x64, 0x29, 0x11, + 0xb6, 0x6e, 0xa6, 0xe0, 0xb8, 0xa3, 0x86, 0xfd, 0x67, 0xec, 0x88, 0x17, 0xa3, 0x57, 0x0d, 0x03, + 0xba, 0xb0, 0x28, 0xf5, 0x0f, 0x73, 0x39, 0xf7, 0xb3, 0x2a, 0x6e, 0x92, 0xbd, 0xf5, 0x80, 0x32, + 0xdb, 0xd9, 0xab, 0xc2, 0x58, 0xf3, 0x03, 0x5d, 0xd7, 0xfc, 0xcf, 0x17, 0xe0, 0x8c, 0x1a, 0x01, + 0x83, 0xaf, 0xfb, 0xf3, 0x3e, 0x06, 0xcf, 0xc3, 0xa8, 0x4b, 0x36, 0x9d, 0x76, 0x33, 0x56, 0x1a, + 0xf1, 0x41, 0xfe, 0x2a, 0x52, 0x49, 0x8a, 0xb1, 0x8e, 0x73, 0x84, 0x61, 0xfb, 0xf1, 0x51, 0x76, + 0xb7, 0xc6, 0x0e, 0x5d, 0xe3, 0x6a, 0xd7, 0x58, 0xb9, 0xbb, 0xe6, 0x49, 0x18, 0xf4, 0x76, 0x28, + 0xaf, 0x55, 0x30, 0x59, 0xa8, 0x15, 0x5a, 0x88, 0x39, 0x0c, 0x7d, 0x02, 0x86, 0xeb, 0xc1, 0xce, + 0x8e, 0xe3, 0xbb, 0xec, 0xca, 0x2b, 0x2d, 0x8c, 0x52, 0x76, 0x6c, 0x91, 0x17, 0x61, 0x09, 0x43, + 0xe7, 0x61, 0xc0, 0x09, 0x1b, 0x5c, 0x2d, 0x51, 0x5a, 0x18, 0xa1, 0x2d, 0xcd, 0x87, 0x8d, 0x08, + 0xb3, 0x52, 0x2a, 0x55, 0xdd, 0x0f, 0xc2, 0x6d, 0xcf, 0x6f, 0x54, 0xbc, 0x50, 0x6c, 0x09, 0x75, + 0x17, 0xde, 0x53, 0x10, 0xac, 0x61, 0xa1, 0x65, 0x18, 0x6c, 0x05, 0x61, 0x1c, 0xcd, 0x0c, 0xb1, + 0xe1, 0x7e, 0x22, 0xe7, 0x20, 0xe2, 0x5f, 0x5b, 0x0d, 0xc2, 0x38, 0xf9, 0x00, 0xfa, 0x2f, 0xc2, + 0xbc, 0x3a, 0xfa, 0x16, 0x28, 0x12, 0x7f, 0x77, 0x66, 0x98, 0x51, 0x99, 0xcd, 0xa2, 0xb2, 0xe4, + 0xef, 0xde, 0x75, 0xc2, 0xe4, 0x94, 0x5e, 0xf2, 0x77, 0x31, 0xad, 0x83, 0x3e, 0x0f, 0x25, 0xb9, + 0xc5, 0x23, 0xa1, 0x31, 0xcb, 0x5c, 0x62, 0xf2, 0x60, 0xc0, 0xe4, 0xbd, 0xb6, 0x17, 0x92, 0x1d, + 0xe2, 0xc7, 0x51, 0x72, 0xa6, 0x49, 0x68, 0x84, 0x13, 0x6a, 0xe8, 0xf3, 0x52, 0x4d, 0xbb, 0x1a, + 0xb4, 0xfd, 0x38, 0x9a, 0x29, 0xb1, 0xee, 0x65, 0x3e, 0xa0, 0xdd, 0x4d, 0xf0, 0xd2, 0x7a, 0x5c, + 0x5e, 0x19, 0x1b, 0xa4, 0x10, 0x86, 0xf1, 0xa6, 0xb7, 0x4b, 0x7c, 0x12, 0x45, 0xd5, 0x30, 0xd8, + 0x20, 0x33, 0xc0, 0x7a, 0x7e, 0x2e, 0xfb, 0x5d, 0x29, 0xd8, 0x20, 0x0b, 0xd3, 0x07, 0xfb, 0xe5, + 0xf1, 0x5b, 0x7a, 0x1d, 0x6c, 0x92, 0x40, 0x77, 0x60, 0x82, 0xca, 0x35, 0x5e, 0x42, 0x74, 0xb4, + 0x17, 0x51, 0x26, 0x7d, 0x60, 0xa3, 0x12, 0x4e, 0x11, 0x41, 0x6f, 0x40, 0xa9, 0xe9, 0x6d, 0x92, + 0xfa, 0x5e, 0xbd, 0x49, 0x66, 0xc6, 0x18, 0xc5, 0xcc, 0x6d, 0x75, 0x4b, 0x22, 0x71, 0xb9, 0x48, + 0xfd, 0xc5, 0x49, 0x75, 0x74, 0x17, 0xce, 0xc6, 0x24, 0xdc, 0xf1, 0x7c, 0x87, 0x6e, 0x07, 0x21, + 0x2f, 0xb0, 0xd7, 0xb9, 0x71, 0xb6, 0xde, 0x2e, 0x8a, 0xa1, 0x3b, 0xbb, 0x9e, 0x89, 0x85, 0x73, + 0x6a, 0xa3, 0xdb, 0x30, 0xc9, 0x76, 0x42, 0xb5, 0xdd, 0x6c, 0x56, 0x83, 0xa6, 0x57, 0xdf, 0x9b, + 0x99, 0x60, 0x04, 0x3f, 0x21, 0xef, 0x85, 0x15, 0x13, 0x7c, 0xb8, 0x5f, 0x86, 0xe4, 0x1f, 0x4e, + 0xd7, 0x46, 0x1b, 0xec, 0x39, 0xa6, 0x1d, 0x7a, 0xf1, 0x1e, 0x5d, 0xbf, 0xe4, 0x41, 0x3c, 0x33, + 0xd9, 0x55, 0x14, 0xd6, 0x51, 0xd5, 0x9b, 0x8d, 0x5e, 0x88, 0xd3, 0x04, 0xe9, 0xd6, 0x8e, 0x62, + 0xd7, 0xf3, 0x67, 0xa6, 0xd8, 0x89, 0xa1, 0x76, 0x46, 0x8d, 0x16, 0x62, 0x0e, 0x63, 0x4f, 0x31, + 0xf4, 0xc7, 0x6d, 0x7a, 0x82, 0x4e, 0x33, 0xc4, 0xe4, 0x29, 0x46, 0x02, 0x70, 0x82, 0x43, 0x99, + 0x9a, 0x38, 0xde, 0x9b, 0x41, 0x0c, 0x55, 0x6d, 0x97, 0xf5, 0xf5, 0xcf, 0x63, 0x5a, 0x8e, 0x6e, + 0xc1, 0x30, 0xf1, 0x77, 0x97, 0xc3, 0x60, 0x67, 0xe6, 0x54, 0xfe, 0x9e, 0x5d, 0xe2, 0x28, 0xfc, + 0x40, 0x4f, 0x04, 0x3c, 0x51, 0x8c, 0x25, 0x09, 0xf4, 0x00, 0x66, 0x32, 0x66, 0x84, 0x4f, 0xc0, + 0x69, 0x36, 0x01, 0xaf, 0x89, 0xba, 0x33, 0xeb, 0x39, 0x78, 0x87, 0x5d, 0x60, 0x38, 0x97, 0x3a, + 0xfa, 0x02, 0x8c, 0xf3, 0x0d, 0xc5, 0xdf, 0x71, 0xa3, 0x99, 0x33, 0xec, 0x6b, 0x2e, 0xe5, 0x6f, + 0x4e, 0x8e, 0xb8, 0x70, 0x46, 0x74, 0x68, 0x5c, 0x2f, 0x8d, 0xb0, 0x49, 0xcd, 0xde, 0x80, 0x09, + 0x75, 0x6e, 0xb1, 0xa5, 0x83, 0xca, 0x30, 0xc8, 0xb8, 0x1d, 0xa1, 0xdf, 0x2a, 0xd1, 0x99, 0x62, + 0x9c, 0x10, 0xe6, 0xe5, 0x6c, 0xa6, 0xbc, 0xf7, 0xc9, 0xc2, 0x5e, 0x4c, 0xb8, 0x54, 0x5d, 0xd4, + 0x66, 0x4a, 0x02, 0x70, 0x82, 0x63, 0xff, 0x3f, 0xce, 0x35, 0x26, 0x87, 0x63, 0x1f, 0xd7, 0xc1, + 0xb3, 0x30, 0xb2, 0x15, 0x44, 0x31, 0xc5, 0x66, 0x6d, 0x0c, 0x26, 0x7c, 0xe2, 0x0d, 0x51, 0x8e, + 0x15, 0x06, 0x7a, 0x15, 0xc6, 0xeb, 0x7a, 0x03, 0xe2, 0x2e, 0x53, 0x43, 0x60, 0xb4, 0x8e, 0x4d, + 0x5c, 0xf4, 0x0a, 0x8c, 0x30, 0x2b, 0x8c, 0x7a, 0xd0, 0x14, 0x4c, 0x96, 0xbc, 0x90, 0x47, 0xaa, + 0xa2, 0xfc, 0x50, 0xfb, 0x8d, 0x15, 0x36, 0xba, 0x0c, 0x43, 0xb4, 0x0b, 0x2b, 0x55, 0x71, 0x8b, + 0x28, 0x55, 0xcd, 0x0d, 0x56, 0x8a, 0x05, 0xd4, 0xfe, 0x9b, 0x05, 0x6d, 0x94, 0xa9, 0x44, 0x4a, + 0x50, 0x15, 0x86, 0xef, 0x3b, 0x5e, 0xec, 0xf9, 0x0d, 0xc1, 0x2e, 0x3c, 0xdd, 0xf5, 0x4a, 0x61, + 0x95, 0xee, 0xf1, 0x0a, 0xfc, 0xd2, 0x13, 0x7f, 0xb0, 0x24, 0x43, 0x29, 0x86, 0x6d, 0xdf, 0xa7, + 0x14, 0x0b, 0xfd, 0x52, 0xc4, 0xbc, 0x02, 0xa7, 0x28, 0xfe, 0x60, 0x49, 0x06, 0xbd, 0x0d, 0x20, + 0x97, 0x25, 0x71, 0x85, 0xf5, 0xc3, 0xb3, 0xbd, 0x89, 0xae, 0xab, 0x3a, 0x0b, 0x13, 0xf4, 0x4a, + 0x4d, 0xfe, 0x63, 0x8d, 0x9e, 0x1d, 0x33, 0xb6, 0xaa, 0xb3, 0x33, 0xe8, 0xdb, 0xe9, 0x49, 0xe0, + 0x84, 0x31, 0x71, 0xe7, 0x63, 0x31, 0x38, 0x9f, 0xec, 0x4f, 0xa6, 0x58, 0xf7, 0x76, 0x88, 0x7e, + 0x6a, 0x08, 0x22, 0x38, 0xa1, 0x67, 0xff, 0x62, 0x11, 0x66, 0xf2, 0xba, 0x4b, 0x17, 0x1d, 0x79, + 0xe0, 0xc5, 0x8b, 0x94, 0x1b, 0xb2, 0xcc, 0x45, 0xb7, 0x24, 0xca, 0xb1, 0xc2, 0xa0, 0xb3, 0x1f, + 0x79, 0x0d, 0x29, 0x12, 0x0e, 0x26, 0xb3, 0x5f, 0x63, 0xa5, 0x58, 0x40, 0x29, 0x5e, 0x48, 0x9c, + 0x48, 0x98, 0xd7, 0x68, 0xab, 0x04, 0xb3, 0x52, 0x2c, 0xa0, 0xba, 0xbe, 0x69, 0xa0, 0x87, 0xbe, + 0xc9, 0x18, 0xa2, 0xc1, 0xe3, 0x1d, 0x22, 0xf4, 0x45, 0x80, 0x4d, 0xcf, 0xf7, 0xa2, 0x2d, 0x46, + 0x7d, 0xe8, 0xc8, 0xd4, 0x15, 0x2f, 0xb5, 0xac, 0xa8, 0x60, 0x8d, 0x22, 0x7a, 0x09, 0x46, 0xd5, + 0x06, 0x5c, 0xa9, 0xb0, 0xb7, 0x46, 0xcd, 0x76, 0x23, 0x39, 0x8d, 0x2a, 0x58, 0xc7, 0xb3, 0xdf, + 0x4d, 0xaf, 0x17, 0xb1, 0x03, 0xb4, 0xf1, 0xb5, 0xfa, 0x1d, 0xdf, 0x42, 0xf7, 0xf1, 0xb5, 0x7f, + 0xad, 0x08, 0x93, 0x46, 0x63, 0xed, 0xa8, 0x8f, 0x33, 0xeb, 0x3a, 0xbd, 0xe7, 0x9c, 0x98, 0x88, + 0xfd, 0x67, 0xf7, 0xde, 0x2a, 0xfa, 0x5d, 0x48, 0x77, 0x00, 0xaf, 0x8f, 0xbe, 0x08, 0xa5, 0xa6, + 0x13, 0x31, 0xdd, 0x15, 0x11, 0xfb, 0xae, 0x1f, 0x62, 0x89, 0x1c, 0xe1, 0x44, 0xb1, 0x76, 0xd5, + 0x70, 0xda, 0x09, 0x49, 0x7a, 0x21, 0x53, 0xde, 0x47, 0xda, 0x6f, 0xa9, 0x4e, 0x50, 0x06, 0x69, + 0x0f, 0x73, 0x18, 0x7a, 0x05, 0xc6, 0x42, 0xc2, 0x56, 0xc5, 0x22, 0x65, 0xe5, 0xd8, 0x32, 0x1b, + 0x4c, 0x78, 0x3e, 0xac, 0xc1, 0xb0, 0x81, 0x99, 0xb0, 0xf2, 0x43, 0x5d, 0x58, 0xf9, 0xa7, 0x61, + 0x98, 0xfd, 0x50, 0x2b, 0x40, 0xcd, 0xc6, 0x0a, 0x2f, 0xc6, 0x12, 0x9e, 0x5e, 0x30, 0x23, 0x7d, + 0x2e, 0x98, 0x4f, 0xc2, 0x44, 0xc5, 0x21, 0x3b, 0x81, 0xbf, 0xe4, 0xbb, 0xad, 0xc0, 0xf3, 0x63, + 0x34, 0x03, 0x03, 0xec, 0x76, 0xe0, 0x7b, 0x7b, 0x80, 0x52, 0xc0, 0x03, 0x94, 0x31, 0xb7, 0x1b, + 0x70, 0xa6, 0x12, 0xdc, 0xf7, 0xef, 0x3b, 0xa1, 0x3b, 0x5f, 0x5d, 0xd1, 0xe4, 0xdc, 0x35, 0x29, + 0x67, 0x71, 0x7b, 0xa8, 0xcc, 0x33, 0x55, 0xab, 0xc9, 0xef, 0xda, 0x65, 0xaf, 0x49, 0x72, 0xb4, + 0x11, 0x7f, 0xa7, 0x60, 0xb4, 0x94, 0xe0, 0xab, 0x07, 0x23, 0x2b, 0xf7, 0xc1, 0xe8, 0x4d, 0x18, + 0xd9, 0xf4, 0x48, 0xd3, 0xc5, 0x64, 0x53, 0x2c, 0xb1, 0xa7, 0xf2, 0x4d, 0x3c, 0x96, 0x29, 0xa6, + 0xd4, 0x3e, 0x71, 0x29, 0x6d, 0x59, 0x54, 0xc6, 0x8a, 0x0c, 0xda, 0x86, 0x29, 0x29, 0x06, 0x48, + 0xa8, 0x58, 0x70, 0x4f, 0x77, 0x93, 0x2d, 0x4c, 0xe2, 0xa7, 0x0f, 0xf6, 0xcb, 0x53, 0x38, 0x45, + 0x06, 0x77, 0x10, 0xa6, 0x62, 0xd9, 0x0e, 0x3d, 0x5a, 0x07, 0xd8, 0xf0, 0x33, 0xb1, 0x8c, 0x49, + 0x98, 0xac, 0xd4, 0xfe, 0x11, 0x0b, 0x1e, 0xeb, 0x18, 0x19, 0x21, 0x69, 0x1f, 0xf3, 0x2c, 0xa4, + 0x25, 0xdf, 0x42, 0x6f, 0xc9, 0xd7, 0xfe, 0x19, 0x0b, 0x4e, 0x2f, 0xed, 0xb4, 0xe2, 0xbd, 0x8a, + 0x67, 0xbe, 0xee, 0xbc, 0x0c, 0x43, 0x3b, 0xc4, 0xf5, 0xda, 0x3b, 0x62, 0xe6, 0xca, 0xf2, 0xf8, + 0x59, 0x65, 0xa5, 0x87, 0xfb, 0xe5, 0xf1, 0x5a, 0x1c, 0x84, 0x4e, 0x83, 0xf0, 0x02, 0x2c, 0xd0, + 0xd9, 0x21, 0xee, 0xbd, 0x4f, 0x6e, 0x79, 0x3b, 0x9e, 0x34, 0xd9, 0xe9, 0xaa, 0x3b, 0x9b, 0x93, + 0x03, 0x3a, 0xf7, 0x66, 0xdb, 0xf1, 0x63, 0x2f, 0xde, 0x13, 0x0f, 0x33, 0x92, 0x08, 0x4e, 0xe8, + 0xd9, 0xdf, 0xb0, 0x60, 0x52, 0xae, 0xfb, 0x79, 0xd7, 0x0d, 0x49, 0x14, 0xa1, 0x59, 0x28, 0x78, + 0x2d, 0xd1, 0x4b, 0x10, 0xbd, 0x2c, 0xac, 0x54, 0x71, 0xc1, 0x6b, 0xa1, 0x2a, 0x94, 0xb8, 0xe5, + 0x4f, 0xb2, 0xb8, 0xfa, 0xb2, 0x1f, 0x62, 0x3d, 0x58, 0x97, 0x35, 0x71, 0x42, 0x44, 0x72, 0x70, + 0xec, 0xcc, 0x2c, 0x9a, 0xaf, 0x5e, 0x37, 0x44, 0x39, 0x56, 0x18, 0xe8, 0x0a, 0x8c, 0xf8, 0x81, + 0xcb, 0x0d, 0xb1, 0xf8, 0xed, 0xc7, 0x96, 0xec, 0x9a, 0x28, 0xc3, 0x0a, 0x6a, 0xff, 0xa0, 0x05, + 0x63, 0xf2, 0xcb, 0xfa, 0x64, 0x26, 0xe9, 0xd6, 0x4a, 0x18, 0xc9, 0x64, 0x6b, 0x51, 0x66, 0x90, + 0x41, 0x0c, 0x1e, 0xb0, 0x78, 0x14, 0x1e, 0xd0, 0xfe, 0xe1, 0x02, 0x4c, 0xc8, 0xee, 0xd4, 0xda, + 0x1b, 0x11, 0x89, 0xd1, 0x3a, 0x94, 0x1c, 0x3e, 0xe4, 0x44, 0xae, 0xd8, 0x27, 0xb3, 0x85, 0x0f, + 0x63, 0x7e, 0x92, 0x6b, 0x79, 0x5e, 0xd6, 0xc6, 0x09, 0x21, 0xd4, 0x84, 0x69, 0x3f, 0x88, 0xd9, + 0x11, 0xad, 0xe0, 0xdd, 0x9e, 0x40, 0xd2, 0xd4, 0xcf, 0x09, 0xea, 0xd3, 0x6b, 0x69, 0x2a, 0xb8, + 0x93, 0x30, 0x5a, 0x92, 0x0a, 0x8f, 0x62, 0xbe, 0xb8, 0xa1, 0xcf, 0x42, 0xb6, 0xbe, 0xc3, 0xfe, + 0x15, 0x0b, 0x4a, 0x12, 0xed, 0x24, 0x5e, 0xbb, 0x56, 0x61, 0x38, 0x62, 0x93, 0x20, 0x87, 0xc6, + 0xee, 0xd6, 0x71, 0x3e, 0x5f, 0xc9, 0xcd, 0xc3, 0xff, 0x47, 0x58, 0xd2, 0x60, 0xfa, 0x6e, 0xd5, + 0xfd, 0x8f, 0x88, 0xbe, 0x5b, 0xf5, 0x27, 0xe7, 0x86, 0xf9, 0x7d, 0xd6, 0x67, 0x4d, 0xac, 0xa5, + 0x0c, 0x52, 0x2b, 0x24, 0x9b, 0xde, 0x83, 0x34, 0x83, 0x54, 0x65, 0xa5, 0x58, 0x40, 0xd1, 0xdb, + 0x30, 0x56, 0x97, 0x8a, 0xce, 0xe4, 0x18, 0xb8, 0xdc, 0x55, 0xe9, 0xae, 0xde, 0x67, 0xb8, 0x91, + 0xf6, 0xa2, 0x56, 0x1f, 0x1b, 0xd4, 0xcc, 0xe7, 0xf6, 0x62, 0xaf, 0xe7, 0xf6, 0x84, 0x6e, 0xfe, + 0xe3, 0xf3, 0x8f, 0x5a, 0x30, 0xc4, 0xd5, 0x65, 0xfd, 0xe9, 0x17, 0xb5, 0xe7, 0xaa, 0x64, 0xec, + 0xee, 0xd2, 0x42, 0xf1, 0xfc, 0x84, 0x56, 0xa1, 0xc4, 0x7e, 0x30, 0xb5, 0x41, 0x31, 0xdf, 0x3a, + 0x9d, 0xb7, 0xaa, 0x77, 0xf0, 0xae, 0xac, 0x86, 0x13, 0x0a, 0xf6, 0x57, 0x8b, 0xf4, 0xa8, 0x4a, + 0x50, 0x8d, 0x1b, 0xdc, 0x7a, 0x74, 0x37, 0x78, 0xe1, 0x51, 0xdd, 0xe0, 0x0d, 0x98, 0xac, 0x6b, + 0x8f, 0x5b, 0xc9, 0x4c, 0x5e, 0xe9, 0xba, 0x48, 0xb4, 0x77, 0x30, 0xae, 0x32, 0x5a, 0x34, 0x89, + 0xe0, 0x34, 0x55, 0xf4, 0xed, 0x30, 0xc6, 0xe7, 0x59, 0xb4, 0xc2, 0x2d, 0x16, 0x3e, 0x91, 0xbf, + 0x5e, 0xf4, 0x26, 0xd8, 0x4a, 0xac, 0x69, 0xd5, 0xb1, 0x41, 0xcc, 0xfe, 0xc5, 0x11, 0x18, 0x5c, + 0xda, 0x25, 0x7e, 0x7c, 0x02, 0x07, 0x52, 0x1d, 0x26, 0x3c, 0x7f, 0x37, 0x68, 0xee, 0x12, 0x97, + 0xc3, 0x8f, 0x72, 0xb9, 0x9e, 0x15, 0xa4, 0x27, 0x56, 0x0c, 0x12, 0x38, 0x45, 0xf2, 0x51, 0x48, + 0x98, 0xd7, 0x61, 0x88, 0xcf, 0xbd, 0x10, 0x2f, 0x33, 0x95, 0xc1, 0x6c, 0x10, 0xc5, 0x2e, 0x48, + 0xa4, 0x5f, 0xae, 0x7d, 0x16, 0xd5, 0xd1, 0xbb, 0x30, 0xb1, 0xe9, 0x85, 0x51, 0x4c, 0x45, 0xc3, + 0x28, 0x76, 0x76, 0x5a, 0x0f, 0x21, 0x51, 0xaa, 0x71, 0x58, 0x36, 0x28, 0xe1, 0x14, 0x65, 0xd4, + 0x80, 0x71, 0x2a, 0xe4, 0x24, 0x4d, 0x0d, 0x1f, 0xb9, 0x29, 0xa5, 0x32, 0xba, 0xa5, 0x13, 0xc2, + 0x26, 0x5d, 0x7a, 0x98, 0xd4, 0x99, 0x50, 0x34, 0xc2, 0x38, 0x0a, 0x75, 0x98, 0x70, 0x69, 0x88, + 0xc3, 0xe8, 0x99, 0xc4, 0xcc, 0x56, 0x4a, 0xe6, 0x99, 0xa4, 0x19, 0xa7, 0xbc, 0x03, 0x25, 0x42, + 0x87, 0x90, 0x12, 0x16, 0x8a, 0xf1, 0xab, 0xfd, 0xf5, 0x75, 0xd5, 0xab, 0x87, 0x81, 0x29, 0xcb, + 0x2f, 0x49, 0x4a, 0x38, 0x21, 0x8a, 0x16, 0x61, 0x28, 0x22, 0xa1, 0x47, 0x22, 0xa1, 0x22, 0xef, + 0x32, 0x8d, 0x0c, 0x8d, 0x5b, 0x7c, 0xf2, 0xdf, 0x58, 0x54, 0xa5, 0xcb, 0xcb, 0x61, 0xd2, 0x10, + 0xd3, 0x8a, 0x6b, 0xcb, 0x6b, 0x9e, 0x95, 0x62, 0x01, 0x45, 0x6f, 0xc0, 0x70, 0x48, 0x9a, 0x4c, + 0x59, 0x34, 0xde, 0xff, 0x22, 0xe7, 0xba, 0x27, 0x5e, 0x0f, 0x4b, 0x02, 0xe8, 0x26, 0xa0, 0x90, + 0x50, 0x1e, 0xc2, 0xf3, 0x1b, 0xca, 0x98, 0x43, 0xe8, 0xba, 0x1f, 0x17, 0xed, 0x9f, 0xc2, 0x09, + 0x86, 0x34, 0xbe, 0xc5, 0x19, 0xd5, 0xd0, 0x75, 0x98, 0x56, 0xa5, 0x2b, 0x7e, 0x14, 0x3b, 0x7e, + 0x9d, 0x30, 0x35, 0x77, 0x29, 0xe1, 0x8a, 0x70, 0x1a, 0x01, 0x77, 0xd6, 0xb1, 0x7f, 0x8a, 0xb2, + 0x33, 0x74, 0xb4, 0x4e, 0x80, 0x17, 0x78, 0xdd, 0xe4, 0x05, 0xce, 0xe5, 0xce, 0x5c, 0x0e, 0x1f, + 0x70, 0x60, 0xc1, 0xa8, 0x36, 0xb3, 0xc9, 0x9a, 0xb5, 0xba, 0xac, 0xd9, 0x36, 0x4c, 0xd1, 0x95, + 0x7e, 0x7b, 0x23, 0x22, 0xe1, 0x2e, 0x71, 0xd9, 0xc2, 0x2c, 0x3c, 0xdc, 0xc2, 0x54, 0xaf, 0xcc, + 0xb7, 0x52, 0x04, 0x71, 0x47, 0x13, 0xe8, 0x65, 0xa9, 0x39, 0x29, 0x1a, 0x46, 0x5a, 0x5c, 0x2b, + 0x72, 0xb8, 0x5f, 0x9e, 0xd2, 0x3e, 0x44, 0xd7, 0x94, 0xd8, 0xef, 0xc8, 0x6f, 0x54, 0xaf, 0xf9, + 0x75, 0xb5, 0x58, 0x52, 0xaf, 0xf9, 0x6a, 0x39, 0xe0, 0x04, 0x87, 0xee, 0x51, 0x2a, 0x82, 0xa4, + 0x5f, 0xf3, 0xa9, 0x80, 0x82, 0x19, 0xc4, 0x7e, 0x01, 0x60, 0xe9, 0x01, 0xa9, 0xf3, 0xa5, 0xae, + 0x3f, 0x40, 0x5a, 0xf9, 0x0f, 0x90, 0xf6, 0x7f, 0xb6, 0x60, 0x62, 0x79, 0xd1, 0x10, 0x13, 0xe7, + 0x00, 0xb8, 0x6c, 0x74, 0xef, 0xde, 0x9a, 0xd4, 0xad, 0x73, 0xf5, 0xa8, 0x2a, 0xc5, 0x1a, 0x06, + 0x3a, 0x07, 0xc5, 0x66, 0xdb, 0x17, 0x22, 0xcb, 0xf0, 0xc1, 0x7e, 0xb9, 0x78, 0xab, 0xed, 0x63, + 0x5a, 0xa6, 0x59, 0x08, 0x16, 0xfb, 0xb6, 0x10, 0xec, 0xe9, 0xa9, 0x87, 0xca, 0x30, 0x78, 0xff, + 0xbe, 0xe7, 0x72, 0x7f, 0x08, 0xa1, 0xf7, 0xbf, 0x77, 0x6f, 0xa5, 0x12, 0x61, 0x5e, 0x6e, 0x7f, + 0xa5, 0x08, 0xb3, 0xcb, 0x4d, 0xf2, 0xe0, 0x03, 0xfa, 0x84, 0xf4, 0x6b, 0xdf, 0x78, 0x34, 0x7e, + 0xf1, 0xa8, 0x36, 0xac, 0xbd, 0xc7, 0x63, 0x13, 0x86, 0xf9, 0x63, 0xb6, 0xf4, 0x10, 0x79, 0x35, + 0xab, 0xf5, 0xfc, 0x01, 0x99, 0xe3, 0x8f, 0xe2, 0xc2, 0xc0, 0x5d, 0xdd, 0xb4, 0xa2, 0x14, 0x4b, + 0xe2, 0xb3, 0x9f, 0x81, 0x31, 0x1d, 0xf3, 0x48, 0xd6, 0xe4, 0x7f, 0xa5, 0x08, 0x53, 0xb4, 0x07, + 0x8f, 0x74, 0x22, 0xee, 0x74, 0x4e, 0xc4, 0x71, 0x5b, 0x14, 0xf7, 0x9e, 0x8d, 0xb7, 0xd3, 0xb3, + 0xf1, 0x7c, 0xde, 0x6c, 0x9c, 0xf4, 0x1c, 0x7c, 0x8f, 0x05, 0xa7, 0x96, 0x9b, 0x41, 0x7d, 0x3b, + 0x65, 0xf5, 0xfb, 0x12, 0x8c, 0xd2, 0x73, 0x3c, 0x32, 0x1c, 0xd2, 0x0c, 0x17, 0x45, 0x01, 0xc2, + 0x3a, 0x9e, 0x56, 0xed, 0xce, 0x9d, 0x95, 0x4a, 0x96, 0x67, 0xa3, 0x00, 0x61, 0x1d, 0xcf, 0xfe, + 0xba, 0x05, 0x17, 0xae, 0x2f, 0x2e, 0x25, 0x4b, 0xb1, 0xc3, 0xb9, 0x92, 0x4a, 0x81, 0xae, 0xd6, + 0x95, 0x44, 0x0a, 0xac, 0xb0, 0x5e, 0x08, 0xe8, 0x47, 0xc5, 0x71, 0xf8, 0x27, 0x2d, 0x38, 0x75, + 0xdd, 0x8b, 0xe9, 0xb5, 0x9c, 0x76, 0xf3, 0xa3, 0xf7, 0x72, 0xe4, 0xc5, 0x41, 0xb8, 0x97, 0x76, + 0xf3, 0xc3, 0x0a, 0x82, 0x35, 0x2c, 0xde, 0xf2, 0xae, 0xc7, 0xcc, 0xa8, 0x0a, 0xa6, 0x2a, 0x0a, + 0x8b, 0x72, 0xac, 0x30, 0xe8, 0x87, 0xb9, 0x5e, 0xc8, 0x44, 0x89, 0x3d, 0x71, 0xc2, 0xaa, 0x0f, + 0xab, 0x48, 0x00, 0x4e, 0x70, 0xec, 0x3f, 0xb4, 0xa0, 0x7c, 0xbd, 0xd9, 0x8e, 0x62, 0x12, 0x6e, + 0x46, 0x39, 0xa7, 0xe3, 0x0b, 0x50, 0x22, 0x52, 0x70, 0x17, 0xbd, 0x56, 0xac, 0xa6, 0x92, 0xe8, + 0xb9, 0xb7, 0xa1, 0xc2, 0xeb, 0xc3, 0x87, 0xe0, 0x68, 0x46, 0xe0, 0xcb, 0x80, 0x88, 0xde, 0x96, + 0xee, 0x7e, 0xc9, 0xfc, 0xb8, 0x96, 0x3a, 0xa0, 0x38, 0xa3, 0x86, 0xfd, 0x23, 0x16, 0x9c, 0x51, + 0x1f, 0xfc, 0x91, 0xfb, 0x4c, 0xfb, 0xe7, 0x0a, 0x30, 0x7e, 0x63, 0x7d, 0xbd, 0x7a, 0x9d, 0xc4, + 0xe2, 0xda, 0xee, 0xad, 0x5b, 0xc7, 0x9a, 0x8a, 0xb0, 0x9b, 0x14, 0xd8, 0x8e, 0xbd, 0xe6, 0x1c, + 0xf7, 0xe2, 0x9f, 0x5b, 0xf1, 0xe3, 0xdb, 0x61, 0x2d, 0x0e, 0x3d, 0xbf, 0x91, 0xa9, 0x54, 0x94, + 0xcc, 0x45, 0x31, 0x8f, 0xb9, 0x40, 0x2f, 0xc0, 0x10, 0x0b, 0x23, 0x20, 0x27, 0xe1, 0x71, 0x25, + 0x44, 0xb1, 0xd2, 0xc3, 0xfd, 0x72, 0xe9, 0x0e, 0x5e, 0xe1, 0x7f, 0xb0, 0x40, 0x45, 0x77, 0x60, + 0x74, 0x2b, 0x8e, 0x5b, 0x37, 0x88, 0xe3, 0x92, 0x50, 0x1e, 0x87, 0x17, 0xb3, 0x8e, 0x43, 0x3a, + 0x08, 0x1c, 0x2d, 0x39, 0x41, 0x92, 0xb2, 0x08, 0xeb, 0x74, 0xec, 0x1a, 0x40, 0x02, 0x3b, 0x26, + 0x85, 0x8a, 0xfd, 0x7b, 0x16, 0x0c, 0x73, 0x8f, 0xce, 0x10, 0xbd, 0x06, 0x03, 0xe4, 0x01, 0xa9, + 0x0b, 0x56, 0x39, 0xb3, 0xc3, 0x09, 0xa7, 0xc5, 0x9f, 0x07, 0xe8, 0x7f, 0xcc, 0x6a, 0xa1, 0x1b, + 0x30, 0x4c, 0x7b, 0x7b, 0x5d, 0xb9, 0xb7, 0x3e, 0x91, 0xf7, 0xc5, 0x6a, 0xda, 0x39, 0x73, 0x26, + 0x8a, 0xb0, 0xac, 0xce, 0x54, 0xdd, 0xf5, 0x56, 0x8d, 0x9e, 0xd8, 0x71, 0x37, 0xc6, 0x62, 0x7d, + 0xb1, 0xca, 0x91, 0x04, 0x35, 0xae, 0xea, 0x96, 0x85, 0x38, 0x21, 0x62, 0xaf, 0x43, 0x89, 0x4e, + 0xea, 0x7c, 0xd3, 0x73, 0xba, 0x6b, 0xd9, 0x9f, 0x81, 0x92, 0xd4, 0x78, 0x47, 0xc2, 0x93, 0x8b, + 0x51, 0x95, 0x0a, 0xf1, 0x08, 0x27, 0x70, 0x7b, 0x13, 0x4e, 0x33, 0x53, 0x07, 0x27, 0xde, 0x32, + 0xf6, 0x58, 0xef, 0xc5, 0xfc, 0xac, 0x90, 0x3c, 0xf9, 0xcc, 0xcc, 0x68, 0xce, 0x12, 0x63, 0x92, + 0x62, 0x22, 0x85, 0xda, 0x7f, 0x30, 0x00, 0x8f, 0xaf, 0xd4, 0xf2, 0x9d, 0x7d, 0x5f, 0x81, 0x31, + 0xce, 0x97, 0xd2, 0xa5, 0xed, 0x34, 0x45, 0xbb, 0xea, 0x21, 0x70, 0x5d, 0x83, 0x61, 0x03, 0x13, + 0x5d, 0x80, 0xa2, 0xf7, 0x9e, 0x9f, 0xb6, 0x3b, 0x5e, 0x79, 0x73, 0x0d, 0xd3, 0x72, 0x0a, 0xa6, + 0x2c, 0x2e, 0xbf, 0x3b, 0x14, 0x58, 0xb1, 0xb9, 0xaf, 0xc3, 0x84, 0x17, 0xd5, 0x23, 0x6f, 0xc5, + 0xa7, 0xe7, 0x8c, 0x76, 0x52, 0x29, 0xad, 0x08, 0xed, 0xb4, 0x82, 0xe2, 0x14, 0xb6, 0x76, 0x91, + 0x0d, 0xf6, 0xcd, 0x26, 0xf7, 0x74, 0x6d, 0xa2, 0x12, 0x40, 0x8b, 0x7d, 0x5d, 0xc4, 0xac, 0xf8, + 0x84, 0x04, 0xc0, 0x3f, 0x38, 0xc2, 0x12, 0x46, 0x45, 0xce, 0xfa, 0x96, 0xd3, 0x9a, 0x6f, 0xc7, + 0x5b, 0x15, 0x2f, 0xaa, 0x07, 0xbb, 0x24, 0xdc, 0x63, 0xda, 0x82, 0x91, 0x44, 0xe4, 0x54, 0x80, + 0xc5, 0x1b, 0xf3, 0x55, 0x8a, 0x89, 0x3b, 0xeb, 0x98, 0x6c, 0x30, 0x1c, 0x07, 0x1b, 0x3c, 0x0f, + 0x93, 0xb2, 0x99, 0x1a, 0x89, 0xd8, 0xa5, 0x38, 0xca, 0x3a, 0xa6, 0x6c, 0x8b, 0x45, 0xb1, 0xea, + 0x56, 0x1a, 0x1f, 0xbd, 0x0c, 0xe3, 0x9e, 0xef, 0xc5, 0x9e, 0x13, 0x07, 0x21, 0x63, 0x29, 0xb8, + 0x62, 0x80, 0x99, 0xee, 0xad, 0xe8, 0x00, 0x6c, 0xe2, 0xd9, 0xff, 0x7d, 0x00, 0xa6, 0xd9, 0xb4, + 0x7d, 0x73, 0x85, 0x7d, 0x64, 0x56, 0xd8, 0x9d, 0xce, 0x15, 0x76, 0x1c, 0xfc, 0xfd, 0x87, 0xb9, + 0xcc, 0xde, 0x85, 0x92, 0x32, 0x7e, 0x96, 0xde, 0x0f, 0x56, 0x8e, 0xf7, 0x43, 0x6f, 0xee, 0x43, + 0xbe, 0x5b, 0x17, 0x33, 0xdf, 0xad, 0xff, 0x9e, 0x05, 0x89, 0x0d, 0x28, 0xba, 0x01, 0xa5, 0x56, + 0xc0, 0xec, 0x2c, 0x42, 0x69, 0xbc, 0xf4, 0x78, 0xe6, 0x45, 0xc5, 0x2f, 0x45, 0x3e, 0x7e, 0x55, + 0x59, 0x03, 0x27, 0x95, 0xd1, 0x02, 0x0c, 0xb7, 0x42, 0x52, 0x8b, 0x99, 0xcf, 0x6f, 0x4f, 0x3a, + 0x7c, 0x8d, 0x70, 0x7c, 0x2c, 0x2b, 0xda, 0x3f, 0x6f, 0x01, 0xf0, 0xa7, 0x61, 0xc7, 0x6f, 0x90, + 0x13, 0x50, 0x77, 0x57, 0x60, 0x20, 0x6a, 0x91, 0x7a, 0x37, 0x0b, 0x98, 0xa4, 0x3f, 0xb5, 0x16, + 0xa9, 0x27, 0x03, 0x4e, 0xff, 0x61, 0x56, 0xdb, 0xfe, 0x5e, 0x80, 0x89, 0x04, 0x6d, 0x25, 0x26, + 0x3b, 0xe8, 0x39, 0xc3, 0x07, 0xf0, 0x5c, 0xca, 0x07, 0xb0, 0xc4, 0xb0, 0x35, 0xcd, 0xea, 0xbb, + 0x50, 0xdc, 0x71, 0x1e, 0x08, 0xd5, 0xd9, 0x33, 0xdd, 0xbb, 0x41, 0xe9, 0xcf, 0xad, 0x3a, 0x0f, + 0xb8, 0x90, 0xf8, 0x8c, 0x5c, 0x20, 0xab, 0xce, 0x83, 0x43, 0x6e, 0xe7, 0xc2, 0x0e, 0xa9, 0x5b, + 0x5e, 0x14, 0x7f, 0xe9, 0xbf, 0x25, 0xff, 0xd9, 0xb2, 0xa3, 0x8d, 0xb0, 0xb6, 0x3c, 0x5f, 0x3c, + 0x94, 0xf6, 0xd5, 0x96, 0xe7, 0xa7, 0xdb, 0xf2, 0xfc, 0x3e, 0xda, 0xf2, 0x7c, 0xf4, 0x3e, 0x0c, + 0x0b, 0xa3, 0x04, 0xe1, 0x73, 0x7f, 0xb5, 0x8f, 0xf6, 0x84, 0x4d, 0x03, 0x6f, 0xf3, 0xaa, 0x14, + 0x82, 0x45, 0x69, 0xcf, 0x76, 0x65, 0x83, 0xe8, 0x6f, 0x5b, 0x30, 0x21, 0x7e, 0x63, 0xf2, 0x5e, + 0x9b, 0x44, 0xb1, 0xe0, 0x3d, 0x3f, 0xdd, 0x7f, 0x1f, 0x44, 0x45, 0xde, 0x95, 0x4f, 0xcb, 0x63, + 0xd6, 0x04, 0xf6, 0xec, 0x51, 0xaa, 0x17, 0xe8, 0x9f, 0x58, 0x70, 0x7a, 0xc7, 0x79, 0xc0, 0x5b, + 0xe4, 0x65, 0xd8, 0x89, 0xbd, 0x40, 0x18, 0xeb, 0xbf, 0xd6, 0xdf, 0xf4, 0x77, 0x54, 0xe7, 0x9d, + 0x94, 0x76, 0xbd, 0xa7, 0xb3, 0x50, 0x7a, 0x76, 0x35, 0xb3, 0x5f, 0xb3, 0x9b, 0x30, 0x22, 0xd7, + 0x5b, 0x86, 0xaa, 0xa1, 0xa2, 0x33, 0xd6, 0x47, 0xb6, 0x09, 0xd1, 0x1d, 0xf1, 0x68, 0x3b, 0x62, + 0xad, 0x3d, 0xd2, 0x76, 0xde, 0x85, 0x31, 0x7d, 0x8d, 0x3d, 0xd2, 0xb6, 0xde, 0x83, 0x53, 0x19, + 0x6b, 0xe9, 0x91, 0x36, 0x79, 0x1f, 0xce, 0xe5, 0xae, 0x8f, 0x47, 0xd9, 0xb0, 0xfd, 0x73, 0x96, + 0x7e, 0x0e, 0x9e, 0xc0, 0x9b, 0xc3, 0xa2, 0xf9, 0xe6, 0x70, 0xb1, 0xfb, 0xce, 0xc9, 0x79, 0x78, + 0x78, 0x5b, 0xef, 0x34, 0x3d, 0xd5, 0xd1, 0x1b, 0x30, 0xd4, 0xa4, 0x25, 0xd2, 0x1a, 0xc6, 0xee, + 0xbd, 0x23, 0x13, 0x5e, 0x8a, 0x95, 0x47, 0x58, 0x50, 0xb0, 0x7f, 0xc9, 0x82, 0x81, 0x13, 0x18, + 0x09, 0x6c, 0x8e, 0xc4, 0x73, 0xb9, 0xa4, 0x45, 0x38, 0xc0, 0x39, 0xec, 0xdc, 0x5f, 0x7a, 0x10, + 0x13, 0x3f, 0x62, 0xa2, 0x62, 0xe6, 0xc0, 0x7c, 0x07, 0x9c, 0xba, 0x15, 0x38, 0xee, 0x82, 0xd3, + 0x74, 0xfc, 0x3a, 0x09, 0x57, 0xfc, 0x46, 0x4f, 0xb3, 0x2c, 0xdd, 0x88, 0xaa, 0xd0, 0xcb, 0x88, + 0xca, 0xde, 0x02, 0xa4, 0x37, 0x20, 0x0c, 0x57, 0x31, 0x0c, 0x7b, 0xbc, 0x29, 0x31, 0xfc, 0x4f, + 0x65, 0x73, 0x77, 0x1d, 0x3d, 0xd3, 0x4c, 0x32, 0x79, 0x01, 0x96, 0x84, 0xec, 0x57, 0x20, 0xd3, + 0x59, 0xad, 0xb7, 0xda, 0xc0, 0xfe, 0x3c, 0x4c, 0xb3, 0x9a, 0x47, 0x14, 0x69, 0xed, 0x94, 0x56, + 0x32, 0x23, 0x32, 0x8d, 0xfd, 0x65, 0x0b, 0x26, 0xd7, 0x52, 0x01, 0x3b, 0x2e, 0xb3, 0x07, 0xd0, + 0x0c, 0x65, 0x78, 0x8d, 0x95, 0x62, 0x01, 0x3d, 0x76, 0x1d, 0xd4, 0x9f, 0x59, 0x90, 0xf8, 0x8f, + 0x9e, 0x00, 0xe3, 0xb5, 0x68, 0x30, 0x5e, 0x99, 0xba, 0x11, 0xd5, 0x9d, 0x3c, 0xbe, 0x0b, 0xdd, + 0x54, 0xc1, 0x12, 0xba, 0xa8, 0x45, 0x12, 0x32, 0xdc, 0xb5, 0x7e, 0xc2, 0x8c, 0xa8, 0x20, 0xc3, + 0x27, 0x30, 0xdb, 0x29, 0x85, 0xfb, 0x11, 0xb1, 0x9d, 0x52, 0xfd, 0xc9, 0xd9, 0xa1, 0x55, 0xad, + 0xcb, 0xec, 0xe4, 0xfa, 0x56, 0x66, 0x0b, 0xef, 0x34, 0xbd, 0xf7, 0x89, 0x8a, 0xf8, 0x52, 0x16, + 0xb6, 0xed, 0xa2, 0xf4, 0x70, 0xbf, 0x3c, 0xae, 0xfe, 0xf1, 0x08, 0x73, 0x49, 0x15, 0xfb, 0x06, + 0x4c, 0xa6, 0x06, 0x0c, 0xbd, 0x04, 0x83, 0xad, 0x2d, 0x27, 0x22, 0x29, 0x7b, 0xd1, 0xc1, 0x2a, + 0x2d, 0x3c, 0xdc, 0x2f, 0x4f, 0xa8, 0x0a, 0xac, 0x04, 0x73, 0x6c, 0xfb, 0x7f, 0x59, 0x30, 0xb0, + 0x16, 0xb8, 0x27, 0xb1, 0x98, 0x5e, 0x37, 0x16, 0xd3, 0xf9, 0xbc, 0xf8, 0x9c, 0xb9, 0xeb, 0x68, + 0x39, 0xb5, 0x8e, 0x2e, 0xe6, 0x52, 0xe8, 0xbe, 0x84, 0x76, 0x60, 0x94, 0x45, 0xfd, 0x14, 0xf6, + 0xab, 0x2f, 0x18, 0x32, 0x40, 0x39, 0x25, 0x03, 0x4c, 0x6a, 0xa8, 0x9a, 0x24, 0xf0, 0x34, 0x0c, + 0x0b, 0x1b, 0xca, 0xb4, 0xd5, 0xbf, 0xc0, 0xc5, 0x12, 0x6e, 0xff, 0x68, 0x11, 0x8c, 0x28, 0xa3, + 0xe8, 0x57, 0x2c, 0x98, 0x0b, 0xb9, 0x1b, 0xa5, 0x5b, 0x69, 0x87, 0x9e, 0xdf, 0xa8, 0xd5, 0xb7, + 0x88, 0xdb, 0x6e, 0x7a, 0x7e, 0x63, 0xa5, 0xe1, 0x07, 0xaa, 0x78, 0xe9, 0x01, 0xa9, 0xb7, 0xd9, + 0x43, 0x48, 0x8f, 0x90, 0xa6, 0xca, 0x46, 0xe9, 0xda, 0xc1, 0x7e, 0x79, 0x0e, 0x1f, 0x89, 0x36, + 0x3e, 0x62, 0x5f, 0xd0, 0xd7, 0x2d, 0xb8, 0xca, 0x83, 0x6f, 0xf6, 0xdf, 0xff, 0x2e, 0x12, 0x53, + 0x55, 0x92, 0x4a, 0x88, 0xac, 0x93, 0x70, 0x67, 0xe1, 0x65, 0x31, 0xa0, 0x57, 0xab, 0x47, 0x6b, + 0x0b, 0x1f, 0xb5, 0x73, 0xf6, 0xbf, 0x29, 0xc2, 0xb8, 0xf0, 0xe0, 0x17, 0xa1, 0x61, 0x5e, 0x32, + 0x96, 0xc4, 0x13, 0xa9, 0x25, 0x31, 0x6d, 0x20, 0x1f, 0x4f, 0x54, 0x98, 0x08, 0xa6, 0x9b, 0x4e, + 0x14, 0xdf, 0x20, 0x4e, 0x18, 0x6f, 0x10, 0x87, 0xdb, 0xee, 0x14, 0x8f, 0x6c, 0x67, 0xa4, 0x54, + 0x34, 0xb7, 0xd2, 0xc4, 0x70, 0x27, 0x7d, 0xb4, 0x0b, 0x88, 0x19, 0x20, 0x85, 0x8e, 0x1f, 0xf1, + 0x6f, 0xf1, 0xc4, 0x9b, 0xc1, 0xd1, 0x5a, 0x9d, 0x15, 0xad, 0xa2, 0x5b, 0x1d, 0xd4, 0x70, 0x46, + 0x0b, 0x9a, 0x61, 0xd9, 0x60, 0xbf, 0x86, 0x65, 0x43, 0x3d, 0x5c, 0x6b, 0x7c, 0x98, 0xea, 0x08, + 0xc2, 0xf0, 0x16, 0x94, 0x94, 0x01, 0xa0, 0x38, 0x74, 0xba, 0xc7, 0x32, 0x49, 0x53, 0xe0, 0x6a, + 0x94, 0xc4, 0xf8, 0x34, 0x21, 0x67, 0xff, 0xd3, 0x82, 0xd1, 0x20, 0x9f, 0xc4, 0x35, 0x18, 0x71, + 0xa2, 0xc8, 0x6b, 0xf8, 0xc4, 0x15, 0x3b, 0xf6, 0xe3, 0x79, 0x3b, 0xd6, 0x68, 0x86, 0x19, 0x61, + 0xce, 0x8b, 0x9a, 0x58, 0xd1, 0x40, 0x37, 0xb8, 0x85, 0xd4, 0xae, 0xe4, 0xf9, 0xfb, 0xa3, 0x06, + 0xd2, 0x86, 0x6a, 0x97, 0x60, 0x51, 0x1f, 0x7d, 0x81, 0x9b, 0xb0, 0xdd, 0xf4, 0x83, 0xfb, 0xfe, + 0xf5, 0x20, 0x90, 0x6e, 0x77, 0xfd, 0x11, 0x9c, 0x96, 0x86, 0x6b, 0xaa, 0x3a, 0x36, 0xa9, 0xf5, + 0x17, 0xa8, 0xe8, 0x3b, 0xe1, 0x14, 0x25, 0x6d, 0x3a, 0xcf, 0x44, 0x88, 0xc0, 0xa4, 0x08, 0x0f, + 0x21, 0xcb, 0xc4, 0xd8, 0x65, 0xb2, 0xf3, 0x66, 0xed, 0x44, 0xe9, 0x77, 0xd3, 0x24, 0x81, 0xd3, + 0x34, 0xed, 0x9f, 0xb0, 0x80, 0x99, 0xfd, 0x9f, 0x00, 0xcb, 0xf0, 0x59, 0x93, 0x65, 0x98, 0xc9, + 0x1b, 0xe4, 0x1c, 0x6e, 0xe1, 0x45, 0xbe, 0xb2, 0xaa, 0x61, 0xf0, 0x60, 0x4f, 0x98, 0x0f, 0xf4, + 0xe6, 0x64, 0xed, 0xff, 0x6b, 0xf1, 0x43, 0x4c, 0x79, 0xe2, 0xa3, 0xef, 0x82, 0x91, 0xba, 0xd3, + 0x72, 0xea, 0x3c, 0x24, 0x76, 0xae, 0x56, 0xc7, 0xa8, 0x34, 0xb7, 0x28, 0x6a, 0x70, 0x2d, 0x85, + 0x0c, 0x33, 0x32, 0x22, 0x8b, 0x7b, 0x6a, 0x26, 0x54, 0x93, 0xb3, 0xdb, 0x30, 0x6e, 0x10, 0x7b, + 0xa4, 0x22, 0xed, 0x77, 0xf1, 0x2b, 0x56, 0x85, 0xc5, 0xd9, 0x81, 0x69, 0x5f, 0xfb, 0x4f, 0x2f, + 0x14, 0x29, 0xa6, 0x7c, 0xbc, 0xd7, 0x25, 0xca, 0x6e, 0x1f, 0xcd, 0xad, 0x21, 0x45, 0x06, 0x77, + 0x52, 0xb6, 0x7f, 0xcc, 0x82, 0xc7, 0x74, 0x44, 0x2d, 0x48, 0x42, 0x2f, 0x3d, 0x71, 0x05, 0x46, + 0x82, 0x16, 0x09, 0x9d, 0x38, 0x08, 0xc5, 0xad, 0x71, 0x45, 0x0e, 0xfa, 0x6d, 0x51, 0x7e, 0x28, + 0x02, 0x4a, 0x4a, 0xea, 0xb2, 0x1c, 0xab, 0x9a, 0x54, 0x8e, 0x61, 0x83, 0x11, 0x89, 0x00, 0x16, + 0xec, 0x0c, 0x60, 0x4f, 0xa6, 0x11, 0x16, 0x10, 0xfb, 0x0f, 0x2c, 0xbe, 0xb0, 0xf4, 0xae, 0xa3, + 0xf7, 0x60, 0x6a, 0xc7, 0x89, 0xeb, 0x5b, 0x4b, 0x0f, 0x5a, 0x21, 0x57, 0x8f, 0xcb, 0x71, 0x7a, + 0xa6, 0xd7, 0x38, 0x69, 0x1f, 0x99, 0x58, 0xe5, 0xad, 0xa6, 0x88, 0xe1, 0x0e, 0xf2, 0x68, 0x03, + 0x46, 0x59, 0x19, 0x33, 0xff, 0x8e, 0xba, 0xb1, 0x06, 0x79, 0xad, 0xa9, 0x57, 0xe7, 0xd5, 0x84, + 0x0e, 0xd6, 0x89, 0xda, 0x5f, 0x2a, 0xf2, 0xdd, 0xce, 0xb8, 0xed, 0xa7, 0x61, 0xb8, 0x15, 0xb8, + 0x8b, 0x2b, 0x15, 0x2c, 0x66, 0x41, 0x5d, 0x23, 0x55, 0x5e, 0x8c, 0x25, 0x1c, 0xbd, 0x0a, 0x40, + 0x1e, 0xc4, 0x24, 0xf4, 0x9d, 0xa6, 0xb2, 0x92, 0x51, 0x76, 0xa1, 0x95, 0x60, 0x2d, 0x88, 0xef, + 0x44, 0xe4, 0x3b, 0x96, 0x14, 0x0a, 0xd6, 0xd0, 0xd1, 0x35, 0x80, 0x56, 0x18, 0xec, 0x7a, 0x2e, + 0xf3, 0x27, 0x2c, 0x9a, 0x36, 0x24, 0x55, 0x05, 0xc1, 0x1a, 0x16, 0x7a, 0x15, 0xc6, 0xdb, 0x7e, + 0xc4, 0x39, 0x14, 0x67, 0x43, 0x84, 0x63, 0x1c, 0x49, 0xac, 0x1b, 0xee, 0xe8, 0x40, 0x6c, 0xe2, + 0xa2, 0x79, 0x18, 0x8a, 0x1d, 0x66, 0x13, 0x31, 0x98, 0x6f, 0xcc, 0xb9, 0x4e, 0x31, 0xf4, 0x80, + 0xcc, 0xb4, 0x02, 0x16, 0x15, 0xd1, 0x5b, 0xd2, 0x39, 0x83, 0x9f, 0xf5, 0xc2, 0x8a, 0xba, 0xbf, + 0x7b, 0x41, 0x73, 0xcd, 0x10, 0xd6, 0xd9, 0x06, 0x2d, 0xfb, 0xeb, 0x25, 0x80, 0x84, 0x1d, 0x47, + 0xef, 0x77, 0x9c, 0x47, 0xcf, 0x76, 0x67, 0xe0, 0x8f, 0xef, 0x30, 0x42, 0xdf, 0x67, 0xc1, 0xa8, + 0xd3, 0x6c, 0x06, 0x75, 0x27, 0x66, 0xa3, 0x5c, 0xe8, 0x7e, 0x1e, 0x8a, 0xf6, 0xe7, 0x93, 0x1a, + 0xbc, 0x0b, 0x2f, 0xc8, 0x85, 0xa7, 0x41, 0x7a, 0xf6, 0x42, 0x6f, 0x18, 0x7d, 0x4a, 0x4a, 0x69, + 0x7c, 0x79, 0xcc, 0xa6, 0xa5, 0xb4, 0x12, 0x3b, 0xfa, 0x35, 0x01, 0x0d, 0xdd, 0x31, 0x22, 0xed, + 0x0d, 0xe4, 0x07, 0x9d, 0x30, 0xb8, 0xd2, 0x5e, 0x41, 0xf6, 0x50, 0x55, 0xf7, 0x26, 0x1b, 0xcc, + 0x8f, 0xcc, 0xa2, 0x89, 0x3f, 0x3d, 0x3c, 0xc9, 0xde, 0x85, 0x49, 0xd7, 0xbc, 0xdb, 0xc5, 0x6a, + 0x7a, 0x2a, 0x8f, 0x6e, 0x8a, 0x15, 0x48, 0x6e, 0xf3, 0x14, 0x00, 0xa7, 0x09, 0xa3, 0x2a, 0xf7, + 0xeb, 0x5b, 0xf1, 0x37, 0x03, 0x61, 0x8d, 0x6f, 0xe7, 0xce, 0xe5, 0x5e, 0x14, 0x93, 0x1d, 0x8a, + 0x99, 0x5c, 0xda, 0x6b, 0xa2, 0x2e, 0x56, 0x54, 0xd0, 0x1b, 0x30, 0xc4, 0x1c, 0x83, 0xa3, 0x99, + 0x91, 0x7c, 0x65, 0xa2, 0x19, 0xd3, 0x22, 0xd9, 0x54, 0xec, 0x6f, 0x84, 0x05, 0x05, 0x74, 0x43, + 0x06, 0xbe, 0x89, 0x56, 0xfc, 0x3b, 0x11, 0x61, 0x81, 0x6f, 0x4a, 0x0b, 0x1f, 0x4f, 0x62, 0xda, + 0xf0, 0xf2, 0xcc, 0xd4, 0x0b, 0x46, 0x4d, 0xca, 0x1c, 0x89, 0xff, 0x32, 0xa3, 0xc3, 0x0c, 0xe4, + 0x77, 0xcf, 0xcc, 0xfa, 0x90, 0x0c, 0xe7, 0x5d, 0x93, 0x04, 0x4e, 0xd3, 0xa4, 0x8c, 0x26, 0xdf, + 0xb9, 0xc2, 0x9e, 0xbf, 0xd7, 0xfe, 0xe7, 0xf2, 0x35, 0xbb, 0x64, 0x78, 0x09, 0x16, 0xf5, 0x4f, + 0xf4, 0xd6, 0x9f, 0xf5, 0x61, 0x2a, 0xbd, 0x45, 0x1f, 0x29, 0x97, 0xf1, 0x7b, 0x03, 0x30, 0x61, + 0x2e, 0x29, 0x74, 0x15, 0x4a, 0x82, 0x88, 0x8a, 0xc2, 0xaa, 0x76, 0xc9, 0xaa, 0x04, 0xe0, 0x04, + 0x87, 0x05, 0xdf, 0x65, 0xd5, 0x35, 0x3b, 0xcc, 0x24, 0xf8, 0xae, 0x82, 0x60, 0x0d, 0x8b, 0xca, + 0x4b, 0x1b, 0x41, 0x10, 0xab, 0x4b, 0x45, 0xad, 0xbb, 0x05, 0x56, 0x8a, 0x05, 0x94, 0x5e, 0x26, + 0xdb, 0x24, 0xf4, 0x49, 0xd3, 0x0c, 0xee, 0xa6, 0x2e, 0x93, 0x9b, 0x3a, 0x10, 0x9b, 0xb8, 0xf4, + 0x96, 0x0c, 0x22, 0xb6, 0x90, 0x85, 0x54, 0x96, 0xd8, 0xb5, 0xd6, 0xb8, 0x8b, 0xbd, 0x84, 0xa3, + 0xcf, 0xc3, 0x63, 0xca, 0x23, 0x1e, 0x73, 0x45, 0xb5, 0x6c, 0x71, 0xc8, 0x50, 0xa2, 0x3c, 0xb6, + 0x98, 0x8d, 0x86, 0xf3, 0xea, 0xa3, 0xd7, 0x61, 0x42, 0x70, 0xee, 0x92, 0xe2, 0xb0, 0x69, 0x3b, + 0x71, 0xd3, 0x80, 0xe2, 0x14, 0xb6, 0x0c, 0x4f, 0xc7, 0x98, 0x67, 0x49, 0x61, 0xa4, 0x33, 0x3c, + 0x9d, 0x0e, 0xc7, 0x1d, 0x35, 0xd0, 0x3c, 0x4c, 0x72, 0xd6, 0xca, 0xf3, 0x1b, 0x7c, 0x4e, 0x84, + 0xbb, 0x8d, 0xda, 0x52, 0xb7, 0x4d, 0x30, 0x4e, 0xe3, 0xa3, 0x57, 0x60, 0xcc, 0x09, 0xeb, 0x5b, + 0x5e, 0x4c, 0xea, 0x71, 0x3b, 0xe4, 0x7e, 0x38, 0x9a, 0xf1, 0xc9, 0xbc, 0x06, 0xc3, 0x06, 0xa6, + 0xfd, 0x3e, 0x9c, 0xca, 0xf0, 0xd4, 0xa3, 0x0b, 0xc7, 0x69, 0x79, 0xf2, 0x9b, 0x52, 0x16, 0xaa, + 0xf3, 0xd5, 0x15, 0xf9, 0x35, 0x1a, 0x16, 0x5d, 0x9d, 0xcc, 0xa3, 0x4f, 0x4b, 0xe0, 0xa2, 0x56, + 0xe7, 0xb2, 0x04, 0xe0, 0x04, 0xc7, 0xfe, 0xdf, 0x05, 0x98, 0xcc, 0x50, 0xbe, 0xb3, 0x24, 0x22, + 0x29, 0xd9, 0x23, 0xc9, 0x19, 0x62, 0x46, 0x3b, 0x2c, 0x1c, 0x21, 0xda, 0x61, 0xb1, 0x57, 0xb4, + 0xc3, 0x81, 0x0f, 0x12, 0xed, 0xd0, 0x1c, 0xb1, 0xc1, 0xbe, 0x46, 0x2c, 0x23, 0x42, 0xe2, 0xd0, + 0x11, 0x23, 0x24, 0x1a, 0x83, 0x3e, 0xdc, 0xc7, 0xa0, 0x7f, 0xb5, 0x00, 0x53, 0x69, 0x23, 0xb9, + 0x13, 0x50, 0xc7, 0xbe, 0x61, 0xa8, 0x63, 0xb3, 0x53, 0xf2, 0xa4, 0x4d, 0xf7, 0xf2, 0x54, 0xb3, + 0x38, 0xa5, 0x9a, 0xfd, 0x64, 0x5f, 0xd4, 0xba, 0xab, 0x69, 0xff, 0x41, 0x01, 0xce, 0xa4, 0xab, + 0x2c, 0x36, 0x1d, 0x6f, 0xe7, 0x04, 0xc6, 0xe6, 0xb6, 0x31, 0x36, 0xcf, 0xf5, 0xf3, 0x35, 0xac, + 0x6b, 0xb9, 0x03, 0x74, 0x2f, 0x35, 0x40, 0x57, 0xfb, 0x27, 0xd9, 0x7d, 0x94, 0xbe, 0x51, 0x84, + 0x8b, 0x99, 0xf5, 0x12, 0x6d, 0xe6, 0xb2, 0xa1, 0xcd, 0xbc, 0x96, 0xd2, 0x66, 0xda, 0xdd, 0x6b, + 0x1f, 0x8f, 0x7a, 0x53, 0xb8, 0x50, 0xb2, 0x88, 0x78, 0x0f, 0xa9, 0xda, 0x34, 0x5c, 0x28, 0x15, + 0x21, 0x6c, 0xd2, 0xfd, 0x8b, 0xa4, 0xd2, 0xfc, 0xf7, 0x16, 0x9c, 0xcb, 0x9c, 0x9b, 0x13, 0x50, + 0x61, 0xad, 0x99, 0x2a, 0xac, 0xa7, 0xfb, 0x5e, 0xad, 0x39, 0x3a, 0xad, 0x5f, 0x1f, 0xc8, 0xf9, + 0x16, 0x26, 0xa0, 0xdf, 0x86, 0x51, 0xa7, 0x5e, 0x27, 0x51, 0xb4, 0x1a, 0xb8, 0x2a, 0x42, 0xdc, + 0x73, 0x4c, 0xce, 0x4a, 0x8a, 0x0f, 0xf7, 0xcb, 0xb3, 0x69, 0x12, 0x09, 0x18, 0xeb, 0x14, 0xcc, + 0xa0, 0x96, 0x85, 0x63, 0x0d, 0x6a, 0x79, 0x0d, 0x60, 0x57, 0x71, 0xeb, 0x69, 0x21, 0x5f, 0xe3, + 0xe3, 0x35, 0x2c, 0xf4, 0x05, 0x18, 0x89, 0xc4, 0x35, 0x2e, 0x96, 0xe2, 0x0b, 0x7d, 0xce, 0x95, + 0xb3, 0x41, 0x9a, 0xa6, 0xaf, 0xbe, 0xd2, 0x87, 0x28, 0x92, 0xe8, 0xdb, 0x60, 0x2a, 0xe2, 0xa1, + 0x60, 0x16, 0x9b, 0x4e, 0xc4, 0xfc, 0x20, 0xc4, 0x2a, 0x64, 0x0e, 0xf8, 0xb5, 0x14, 0x0c, 0x77, + 0x60, 0xa3, 0x65, 0xf9, 0x51, 0x2c, 0x6e, 0x0d, 0x5f, 0x98, 0x97, 0x93, 0x0f, 0x12, 0x29, 0xcc, + 0x4e, 0xa7, 0x87, 0x9f, 0x0d, 0xbc, 0x56, 0x13, 0x7d, 0x01, 0x80, 0x2e, 0x1f, 0xa1, 0x4b, 0x18, + 0xce, 0x3f, 0x3c, 0xe9, 0xa9, 0xe2, 0x66, 0x5a, 0x7e, 0x32, 0xe7, 0xc5, 0x8a, 0x22, 0x82, 0x35, + 0x82, 0xf6, 0x57, 0x07, 0xe0, 0xf1, 0x2e, 0x67, 0x24, 0x9a, 0x37, 0x9f, 0x40, 0x9f, 0x49, 0x0b, + 0xd7, 0xb3, 0x99, 0x95, 0x0d, 0x69, 0x3b, 0xb5, 0x14, 0x0b, 0x1f, 0x78, 0x29, 0xfe, 0x80, 0xa5, + 0xa9, 0x3d, 0xb8, 0x31, 0xdf, 0x67, 0x8f, 0x78, 0xf6, 0x1f, 0xa3, 0x1e, 0x64, 0x33, 0x43, 0x99, + 0x70, 0xad, 0xef, 0xee, 0xf4, 0xad, 0x5d, 0x38, 0x59, 0xe5, 0xef, 0x97, 0x2c, 0x78, 0x22, 0xb3, + 0xbf, 0x86, 0xc9, 0xc6, 0x55, 0x28, 0xd5, 0x69, 0xa1, 0xe6, 0xab, 0x96, 0x38, 0xf1, 0x4a, 0x00, + 0x4e, 0x70, 0x0c, 0xcb, 0x8c, 0x42, 0x4f, 0xcb, 0x8c, 0x7f, 0x6d, 0x41, 0xc7, 0xfe, 0x38, 0x81, + 0x83, 0x7a, 0xc5, 0x3c, 0xa8, 0x3f, 0xde, 0xcf, 0x5c, 0xe6, 0x9c, 0xd1, 0x7f, 0x34, 0x09, 0x67, + 0x73, 0x7c, 0x35, 0x76, 0x61, 0xba, 0x51, 0x27, 0xa6, 0x17, 0xa0, 0xf8, 0x98, 0x4c, 0x87, 0xc9, + 0xae, 0x2e, 0x83, 0x2c, 0x1f, 0xd1, 0x74, 0x07, 0x0a, 0xee, 0x6c, 0x02, 0x7d, 0xc9, 0x82, 0xd3, + 0xce, 0xfd, 0xa8, 0x23, 0x81, 0xa9, 0x58, 0x33, 0x2f, 0x66, 0x2a, 0x41, 0x7a, 0x24, 0x3c, 0xe5, + 0x09, 0x9a, 0xb2, 0xb0, 0x70, 0x66, 0x5b, 0x08, 0x8b, 0x98, 0xa1, 0x94, 0x9d, 0xef, 0xe2, 0xa7, + 0x9a, 0xe5, 0x54, 0xc3, 0x8f, 0x6c, 0x09, 0xc1, 0x8a, 0x0e, 0x7a, 0x07, 0x4a, 0x0d, 0xe9, 0xe9, + 0x96, 0x71, 0x25, 0x24, 0x03, 0xd9, 0xdd, 0xff, 0x8f, 0x3f, 0x50, 0x2a, 0x24, 0x9c, 0x10, 0x45, + 0xaf, 0x43, 0xd1, 0xdf, 0x8c, 0xba, 0xe5, 0x38, 0x4a, 0xd9, 0x34, 0x71, 0x6f, 0xf0, 0xb5, 0xe5, + 0x1a, 0xa6, 0x15, 0xd1, 0x0d, 0x28, 0x86, 0x1b, 0xae, 0xd0, 0xe0, 0x65, 0x9e, 0xe1, 0x78, 0xa1, + 0x92, 0xd3, 0x2b, 0x46, 0x09, 0x2f, 0x54, 0x30, 0x25, 0x81, 0xaa, 0x30, 0xc8, 0x1c, 0x1c, 0xc4, + 0x7d, 0x90, 0xc9, 0xf9, 0x76, 0x71, 0x14, 0xe2, 0x2e, 0xe3, 0x0c, 0x01, 0x73, 0x42, 0x68, 0x1d, + 0x86, 0xea, 0x2c, 0x1f, 0x8e, 0x08, 0x58, 0xfd, 0xa9, 0x4c, 0x5d, 0x5d, 0x97, 0x44, 0x41, 0x42, + 0x75, 0xc5, 0x30, 0xb0, 0xa0, 0xc5, 0xa8, 0x92, 0xd6, 0xd6, 0x66, 0x24, 0xf2, 0xb7, 0x65, 0x53, + 0xed, 0x92, 0xff, 0x4a, 0x50, 0x65, 0x18, 0x58, 0xd0, 0x42, 0x9f, 0x81, 0xc2, 0x66, 0x5d, 0xf8, + 0x3f, 0x64, 0x2a, 0xed, 0x4c, 0x87, 0xfe, 0x85, 0xa1, 0x83, 0xfd, 0x72, 0x61, 0x79, 0x11, 0x17, + 0x36, 0xeb, 0x68, 0x0d, 0x86, 0x37, 0xb9, 0x0b, 0xb0, 0xd0, 0xcb, 0x3d, 0x95, 0xed, 0x9d, 0xdc, + 0xe1, 0x25, 0xcc, 0xed, 0xf6, 0x05, 0x00, 0x4b, 0x22, 0x2c, 0x04, 0xa7, 0x72, 0x65, 0x16, 0xb1, + 0xa8, 0xe7, 0x8e, 0xe6, 0x7e, 0xce, 0xef, 0xe7, 0xc4, 0x21, 0x1a, 0x6b, 0x14, 0xe9, 0xaa, 0x76, + 0x64, 0x12, 0x4d, 0x11, 0xab, 0x23, 0x73, 0x55, 0xf7, 0xc8, 0x2f, 0xca, 0x57, 0xb5, 0x42, 0xc2, + 0x09, 0x51, 0xb4, 0x0d, 0xe3, 0xbb, 0x51, 0x6b, 0x8b, 0xc8, 0x2d, 0xcd, 0x42, 0x77, 0xe4, 0x5c, + 0x61, 0x77, 0x05, 0xa2, 0x17, 0xc6, 0x6d, 0xa7, 0xd9, 0x71, 0x0a, 0xb1, 0x57, 0xed, 0xbb, 0x3a, + 0x31, 0x6c, 0xd2, 0xa6, 0xc3, 0xff, 0x5e, 0x3b, 0xd8, 0xd8, 0x8b, 0x89, 0x08, 0x5e, 0x9d, 0x39, + 0xfc, 0x6f, 0x72, 0x94, 0xce, 0xe1, 0x17, 0x00, 0x2c, 0x89, 0xa0, 0xbb, 0x62, 0x78, 0xd8, 0xe9, + 0x39, 0x95, 0x1f, 0x4c, 0x29, 0x33, 0x8b, 0xad, 0x36, 0x28, 0xec, 0xb4, 0x4c, 0x48, 0xb1, 0x53, + 0xb2, 0xb5, 0x15, 0xc4, 0x81, 0x9f, 0x3a, 0xa1, 0xa7, 0xf3, 0x4f, 0xc9, 0x6a, 0x06, 0x7e, 0xe7, + 0x29, 0x99, 0x85, 0x85, 0x33, 0xdb, 0x42, 0x2e, 0x4c, 0xb4, 0x82, 0x30, 0xbe, 0x1f, 0x84, 0x72, + 0x7d, 0xa1, 0x2e, 0x7a, 0x05, 0x03, 0x53, 0xb4, 0xc8, 0x82, 0xa9, 0x9b, 0x10, 0x9c, 0xa2, 0x89, + 0x3e, 0x07, 0xc3, 0x51, 0xdd, 0x69, 0x92, 0x95, 0xdb, 0x33, 0xa7, 0xf2, 0xaf, 0x9f, 0x1a, 0x47, + 0xc9, 0x59, 0x5d, 0x6c, 0x72, 0x04, 0x0a, 0x96, 0xe4, 0xd0, 0x32, 0x0c, 0xb2, 0x8c, 0x08, 0x2c, + 0xee, 0x76, 0x4e, 0x4c, 0xa8, 0x0e, 0x0b, 0x53, 0x7e, 0x36, 0xb1, 0x62, 0xcc, 0xab, 0xd3, 0x3d, + 0x20, 0xd8, 0xeb, 0x20, 0x9a, 0x39, 0x93, 0xbf, 0x07, 0x04, 0x57, 0x7e, 0xbb, 0xd6, 0x6d, 0x0f, + 0x28, 0x24, 0x9c, 0x10, 0xa5, 0x27, 0x33, 0x3d, 0x4d, 0xcf, 0x76, 0x31, 0x68, 0xc9, 0x3d, 0x4b, + 0xd9, 0xc9, 0x4c, 0x4f, 0x52, 0x4a, 0xc2, 0xfe, 0x9d, 0xe1, 0x4e, 0x9e, 0x85, 0x09, 0x64, 0x7f, + 0xd5, 0xea, 0x78, 0xab, 0xfb, 0x74, 0xbf, 0xfa, 0xa1, 0x63, 0xe4, 0x56, 0xbf, 0x64, 0xc1, 0xd9, + 0x56, 0xe6, 0x87, 0x08, 0x06, 0xa0, 0x3f, 0x35, 0x13, 0xff, 0x74, 0x15, 0x1b, 0x3f, 0x1b, 0x8e, + 0x73, 0x5a, 0x4a, 0x4b, 0x04, 0xc5, 0x0f, 0x2c, 0x11, 0xac, 0xc2, 0x08, 0x63, 0x32, 0x7b, 0xe4, + 0x87, 0x4b, 0x0b, 0x46, 0x8c, 0x95, 0x58, 0x14, 0x15, 0xb1, 0x22, 0x81, 0x7e, 0xd0, 0x82, 0x0b, + 0xe9, 0xae, 0x63, 0xc2, 0xc0, 0x22, 0x92, 0x3c, 0x97, 0x05, 0x97, 0xc5, 0xf7, 0x5f, 0xa8, 0x76, + 0x43, 0x3e, 0xec, 0x85, 0x80, 0xbb, 0x37, 0x86, 0x2a, 0x19, 0xc2, 0xe8, 0x90, 0xa9, 0x80, 0xef, + 0x43, 0x20, 0x7d, 0x11, 0xc6, 0x76, 0x82, 0xb6, 0x1f, 0x0b, 0xfb, 0x17, 0xe1, 0xb1, 0xc8, 0x1e, + 0x9c, 0x57, 0xb5, 0x72, 0x6c, 0x60, 0xa5, 0xc4, 0xd8, 0x91, 0x87, 0x16, 0x63, 0xdf, 0x4e, 0x25, + 0x94, 0x2f, 0xe5, 0x47, 0x2c, 0x14, 0x12, 0xff, 0x11, 0xd2, 0xca, 0x9f, 0xac, 0x6c, 0xf4, 0x53, + 0x56, 0x06, 0x53, 0xcf, 0xa5, 0xe5, 0xd7, 0x4c, 0x69, 0xf9, 0x72, 0x5a, 0x5a, 0xee, 0x50, 0xbe, + 0x1a, 0x82, 0x72, 0xff, 0x61, 0xaf, 0xfb, 0x8d, 0x23, 0x67, 0x37, 0xe1, 0x52, 0xaf, 0x6b, 0x89, + 0x19, 0x42, 0xb9, 0xea, 0xa9, 0x2d, 0x31, 0x84, 0x72, 0x57, 0x2a, 0x98, 0x41, 0xfa, 0x0d, 0x34, + 0x62, 0xff, 0x4f, 0x0b, 0x8a, 0xd5, 0xc0, 0x3d, 0x01, 0x65, 0xf2, 0x67, 0x0d, 0x65, 0xf2, 0xe3, + 0x39, 0x89, 0xfe, 0x73, 0x55, 0xc7, 0x4b, 0x29, 0xd5, 0xf1, 0x85, 0x3c, 0x02, 0xdd, 0x15, 0xc5, + 0x3f, 0x5e, 0x84, 0xd1, 0x6a, 0xe0, 0x2a, 0x2b, 0xe4, 0x5f, 0x7f, 0x18, 0x2b, 0xe4, 0xdc, 0xb0, + 0xb0, 0x1a, 0x65, 0x66, 0x3f, 0x25, 0x9d, 0xf0, 0xfe, 0x9c, 0x19, 0x23, 0xdf, 0x23, 0x5e, 0x63, + 0x2b, 0x26, 0x6e, 0xfa, 0x73, 0x4e, 0xce, 0x18, 0xf9, 0x7f, 0x58, 0x30, 0x99, 0x6a, 0x1d, 0x35, + 0x61, 0xbc, 0xa9, 0x6b, 0x02, 0xc5, 0x3a, 0x7d, 0x28, 0x25, 0xa2, 0x30, 0xe6, 0xd4, 0x8a, 0xb0, + 0x49, 0x1c, 0xcd, 0x01, 0xa8, 0x97, 0x3a, 0xa9, 0x01, 0x63, 0x5c, 0xbf, 0x7a, 0xca, 0x8b, 0xb0, + 0x86, 0x81, 0x5e, 0x82, 0xd1, 0x38, 0x68, 0x05, 0xcd, 0xa0, 0xb1, 0x77, 0x93, 0xc8, 0xd0, 0x36, + 0xca, 0x44, 0x6b, 0x3d, 0x01, 0x61, 0x1d, 0xcf, 0xfe, 0xc9, 0x22, 0xff, 0x50, 0x3f, 0xf6, 0xbe, + 0xb9, 0x26, 0x3f, 0xda, 0x6b, 0xf2, 0x1b, 0x16, 0x4c, 0xd1, 0xd6, 0x99, 0xb9, 0x88, 0xbc, 0x6c, + 0x55, 0xfa, 0x1d, 0xab, 0x4b, 0xfa, 0x9d, 0xcb, 0xf4, 0xec, 0x72, 0x83, 0x76, 0x2c, 0x34, 0x68, + 0xda, 0xe1, 0x44, 0x4b, 0xb1, 0x80, 0x0a, 0x3c, 0x12, 0x86, 0xc2, 0x07, 0x4a, 0xc7, 0x23, 0x61, + 0x88, 0x05, 0x54, 0x66, 0xe7, 0x19, 0xc8, 0xc9, 0xce, 0xc3, 0x02, 0xf5, 0x09, 0xc3, 0x02, 0xc1, + 0xf6, 0x68, 0x81, 0xfa, 0xa4, 0xc5, 0x41, 0x82, 0x63, 0xff, 0x5c, 0x11, 0xc6, 0xaa, 0x81, 0x9b, + 0xbc, 0x95, 0xbd, 0x68, 0xbc, 0x95, 0x5d, 0x4a, 0xbd, 0x95, 0x4d, 0xe9, 0xb8, 0xdf, 0x7c, 0x19, + 0xfb, 0xb0, 0x5e, 0xc6, 0xfe, 0x95, 0xc5, 0x66, 0xad, 0xb2, 0x56, 0x13, 0xd9, 0x81, 0x9f, 0x87, + 0x51, 0x76, 0x20, 0x31, 0xa7, 0x3b, 0xf9, 0x80, 0xc4, 0x02, 0xef, 0xaf, 0x25, 0xc5, 0x58, 0xc7, + 0x41, 0x57, 0x60, 0x24, 0x22, 0x4e, 0x58, 0xdf, 0x52, 0x67, 0x9c, 0x78, 0x5e, 0xe1, 0x65, 0x58, + 0x41, 0xd1, 0x9b, 0x49, 0x8c, 0xb8, 0x62, 0x7e, 0x9e, 0x5b, 0xbd, 0x3f, 0x7c, 0x8b, 0xe4, 0x07, + 0x86, 0xb3, 0xef, 0x01, 0xea, 0xc4, 0xef, 0x23, 0x38, 0x52, 0xd9, 0x0c, 0x8e, 0x54, 0xea, 0x08, + 0x8c, 0xf4, 0xa7, 0x16, 0x4c, 0x54, 0x03, 0x97, 0x6e, 0xdd, 0xbf, 0x48, 0xfb, 0x54, 0x0f, 0x90, + 0x39, 0xd4, 0x25, 0x40, 0xe6, 0x3f, 0xb4, 0x60, 0xb8, 0x1a, 0xb8, 0x27, 0xa0, 0x77, 0x7f, 0xcd, + 0xd4, 0xbb, 0x3f, 0x96, 0xb3, 0x24, 0x72, 0x54, 0xed, 0xbf, 0x50, 0x84, 0x71, 0xda, 0xcf, 0xa0, + 0x21, 0x67, 0xc9, 0x18, 0x11, 0xab, 0x8f, 0x11, 0xa1, 0x6c, 0x6e, 0xd0, 0x6c, 0x06, 0xf7, 0xd3, + 0x33, 0xb6, 0xcc, 0x4a, 0xb1, 0x80, 0xa2, 0x67, 0x61, 0xa4, 0x15, 0x92, 0x5d, 0x2f, 0x10, 0xfc, + 0xa3, 0xf6, 0x8a, 0x51, 0x15, 0xe5, 0x58, 0x61, 0x50, 0xb9, 0x2b, 0xf2, 0xfc, 0x3a, 0x91, 0x49, + 0xb6, 0x07, 0x58, 0x1e, 0x2e, 0x1e, 0xf9, 0x5a, 0x2b, 0xc7, 0x06, 0x16, 0xba, 0x07, 0x25, 0xf6, + 0x9f, 0x9d, 0x28, 0x47, 0xcf, 0x1b, 0x24, 0xd2, 0x4d, 0x08, 0x02, 0x38, 0xa1, 0x85, 0xae, 0x01, + 0xc4, 0x32, 0x3a, 0x72, 0x24, 0x62, 0xdc, 0x28, 0x5e, 0x5b, 0xc5, 0x4d, 0x8e, 0xb0, 0x86, 0x85, + 0x9e, 0x81, 0x52, 0xec, 0x78, 0xcd, 0x5b, 0x9e, 0x4f, 0x22, 0xa6, 0x72, 0x2e, 0xca, 0x6c, 0x12, + 0xa2, 0x10, 0x27, 0x70, 0xca, 0xeb, 0x30, 0x07, 0x70, 0x9e, 0x75, 0x6c, 0x84, 0x61, 0x33, 0x5e, + 0xe7, 0x96, 0x2a, 0xc5, 0x1a, 0x86, 0xfd, 0x0a, 0x9c, 0xa9, 0x06, 0x6e, 0x35, 0x08, 0xe3, 0xe5, + 0x20, 0xbc, 0xef, 0x84, 0xae, 0x9c, 0xbf, 0xb2, 0x4c, 0x6c, 0x40, 0xcf, 0x9e, 0x41, 0xbe, 0x33, + 0x8d, 0x94, 0x05, 0x2f, 0x30, 0x6e, 0xe7, 0x88, 0x4e, 0x1d, 0x75, 0x76, 0xef, 0xaa, 0x04, 0x83, + 0xd7, 0x9d, 0x98, 0xa0, 0xdb, 0x2c, 0x29, 0x59, 0x72, 0x05, 0x89, 0xea, 0x4f, 0x6b, 0x49, 0xc9, + 0x12, 0x60, 0xe6, 0x9d, 0x65, 0xd6, 0xb7, 0x7f, 0x66, 0x80, 0x9d, 0x46, 0xa9, 0x7c, 0x7b, 0xe8, + 0x8b, 0x30, 0x11, 0x91, 0x5b, 0x9e, 0xdf, 0x7e, 0x20, 0x85, 0xf0, 0x2e, 0x6e, 0x39, 0xb5, 0x25, + 0x1d, 0x93, 0xab, 0xf2, 0xcc, 0x32, 0x9c, 0xa2, 0x46, 0xe7, 0x29, 0x6c, 0xfb, 0xf3, 0xd1, 0x9d, + 0x88, 0x84, 0x22, 0xdf, 0x1b, 0x9b, 0x27, 0x2c, 0x0b, 0x71, 0x02, 0xa7, 0xeb, 0x92, 0xfd, 0x59, + 0x0b, 0x7c, 0x1c, 0x04, 0xb1, 0x5c, 0xc9, 0x2c, 0x63, 0x90, 0x56, 0x8e, 0x0d, 0x2c, 0xb4, 0x0c, + 0x28, 0x6a, 0xb7, 0x5a, 0x4d, 0xf6, 0xb0, 0xef, 0x34, 0xaf, 0x87, 0x41, 0xbb, 0xc5, 0x5f, 0x3d, + 0x8b, 0x3c, 0x30, 0x61, 0xad, 0x03, 0x8a, 0x33, 0x6a, 0xd0, 0xd3, 0x67, 0x33, 0x62, 0xbf, 0xd9, + 0xea, 0x2e, 0x0a, 0xf5, 0x7a, 0x8d, 0x15, 0x61, 0x09, 0xa3, 0x8b, 0x89, 0x35, 0xcf, 0x31, 0x87, + 0x92, 0xc5, 0x84, 0x55, 0x29, 0xd6, 0x30, 0xd0, 0x12, 0x0c, 0x47, 0x7b, 0x51, 0x3d, 0x16, 0x11, + 0x99, 0x72, 0x32, 0x77, 0xd6, 0x18, 0x8a, 0x96, 0x4d, 0x82, 0x57, 0xc1, 0xb2, 0x2e, 0xda, 0x81, + 0x89, 0xfb, 0x9e, 0xef, 0x06, 0xf7, 0x23, 0x39, 0x51, 0x23, 0xf9, 0xaa, 0xd1, 0x7b, 0x1c, 0x33, + 0x35, 0xd9, 0xc6, 0xbc, 0xdd, 0x33, 0x88, 0xe1, 0x14, 0x71, 0xfb, 0xbb, 0xd8, 0xdd, 0xcb, 0x92, + 0x91, 0xc5, 0xed, 0x90, 0xa0, 0x1d, 0x18, 0x6f, 0xb1, 0x15, 0x26, 0x42, 0x65, 0x8b, 0x65, 0xf2, + 0x62, 0x9f, 0x42, 0xf4, 0x7d, 0x7a, 0xae, 0x29, 0x25, 0x17, 0x93, 0x4e, 0xaa, 0x3a, 0x39, 0x6c, + 0x52, 0xb7, 0xbf, 0x8a, 0xd8, 0x11, 0x5f, 0xe3, 0x92, 0xf1, 0xb0, 0xb0, 0x64, 0x16, 0x62, 0xc0, + 0x6c, 0xbe, 0x8a, 0x26, 0x19, 0x40, 0x61, 0x0d, 0x8d, 0x65, 0x5d, 0xf4, 0x26, 0x7b, 0x14, 0xe7, + 0xe7, 0x6a, 0xaf, 0x9c, 0xd0, 0x1c, 0xcb, 0x78, 0xff, 0x16, 0x15, 0xb1, 0x46, 0x04, 0xdd, 0x82, + 0x71, 0x91, 0xbb, 0x4a, 0xe8, 0xe0, 0x8a, 0x86, 0x8e, 0x65, 0x1c, 0xeb, 0xc0, 0xc3, 0x74, 0x01, + 0x36, 0x2b, 0xa3, 0x06, 0x5c, 0xd0, 0x12, 0x39, 0x5e, 0x0f, 0x1d, 0xf6, 0x50, 0xea, 0xb1, 0x3d, + 0xab, 0x1d, 0xd3, 0x4f, 0x1c, 0xec, 0x97, 0x2f, 0xac, 0x77, 0x43, 0xc4, 0xdd, 0xe9, 0xa0, 0xdb, + 0x70, 0x86, 0x3b, 0x0c, 0x56, 0x88, 0xe3, 0x36, 0x3d, 0x5f, 0xdd, 0x03, 0x7c, 0xd9, 0x9f, 0x3b, + 0xd8, 0x2f, 0x9f, 0x99, 0xcf, 0x42, 0xc0, 0xd9, 0xf5, 0xd0, 0x6b, 0x50, 0x72, 0xfd, 0x48, 0x8c, + 0xc1, 0x90, 0x91, 0xa3, 0xb4, 0x54, 0x59, 0xab, 0xa9, 0xef, 0x4f, 0xfe, 0xe0, 0xa4, 0x02, 0x6a, + 0x70, 0x3d, 0x9c, 0x12, 0x7b, 0x87, 0xf3, 0xf3, 0xd1, 0x8b, 0x25, 0x61, 0xb8, 0x0c, 0x71, 0x05, + 0xb4, 0x32, 0xb9, 0x35, 0xbc, 0x89, 0x0c, 0xc2, 0xe8, 0x0d, 0x40, 0x94, 0x2f, 0xf4, 0xea, 0x64, + 0xbe, 0xce, 0x22, 0x96, 0x33, 0xb5, 0xe5, 0x88, 0xe1, 0xa2, 0x81, 0x6a, 0x1d, 0x18, 0x38, 0xa3, + 0x16, 0xba, 0x41, 0xcf, 0x4d, 0xbd, 0x54, 0x98, 0x0e, 0x4b, 0x59, 0x62, 0xa6, 0x42, 0x5a, 0x21, + 0xa9, 0x3b, 0x31, 0x71, 0x4d, 0x8a, 0x38, 0x55, 0x8f, 0x5e, 0xdd, 0x2a, 0x79, 0x11, 0x98, 0x51, + 0x3a, 0x3a, 0x13, 0x18, 0x51, 0x31, 0x7c, 0x2b, 0x88, 0xe2, 0x35, 0x12, 0xdf, 0x0f, 0xc2, 0x6d, + 0x11, 0x14, 0x2d, 0x89, 0xcf, 0x99, 0x80, 0xb0, 0x8e, 0x47, 0xd9, 0x6e, 0xf6, 0x2a, 0xbd, 0x52, + 0x61, 0x0f, 0x82, 0x23, 0xc9, 0x3e, 0xb9, 0xc1, 0x8b, 0xb1, 0x84, 0x4b, 0xd4, 0x95, 0xea, 0x22, + 0x7b, 0xdc, 0x4b, 0xa1, 0xae, 0x54, 0x17, 0xb1, 0x84, 0x23, 0xd2, 0x99, 0xff, 0x75, 0x22, 0x5f, + 0x89, 0xda, 0x79, 0xfb, 0xf4, 0x99, 0x02, 0xd6, 0x87, 0x29, 0x95, 0x79, 0x96, 0x47, 0x8b, 0x8b, + 0x66, 0x26, 0xd9, 0x22, 0xe9, 0x3f, 0xd4, 0x9c, 0x52, 0x4b, 0xaf, 0xa4, 0x28, 0xe1, 0x0e, 0xda, + 0x46, 0xdc, 0x94, 0xa9, 0x9e, 0xc9, 0xa7, 0xae, 0x42, 0x29, 0x6a, 0x6f, 0xb8, 0xc1, 0x8e, 0xe3, + 0xf9, 0xec, 0x2d, 0x4e, 0xe3, 0xe9, 0x6a, 0x12, 0x80, 0x13, 0x1c, 0xb4, 0x0c, 0x23, 0x8e, 0xd4, + 0x39, 0xa3, 0xfc, 0x20, 0x09, 0x4a, 0xd3, 0xcc, 0xfd, 0x86, 0xa5, 0x96, 0x59, 0xd5, 0x45, 0xaf, + 0xc2, 0xb8, 0x70, 0x13, 0xe3, 0xa1, 0x23, 0xd8, 0x5b, 0x99, 0xe6, 0x07, 0x50, 0xd3, 0x81, 0xd8, + 0xc4, 0x45, 0x5f, 0x80, 0x09, 0x4a, 0x25, 0x39, 0xd8, 0x66, 0x4e, 0xf7, 0x73, 0x22, 0x6a, 0x49, + 0x45, 0xf4, 0xca, 0x38, 0x45, 0x0c, 0xb9, 0x70, 0xde, 0x69, 0xc7, 0x01, 0xd3, 0xdb, 0x9b, 0xeb, + 0x7f, 0x3d, 0xd8, 0x26, 0x3e, 0x7b, 0x32, 0x1b, 0x59, 0xb8, 0x74, 0xb0, 0x5f, 0x3e, 0x3f, 0xdf, + 0x05, 0x0f, 0x77, 0xa5, 0x82, 0xee, 0xc0, 0x68, 0x1c, 0x34, 0x99, 0x45, 0x3e, 0xbd, 0x10, 0xcf, + 0xe6, 0xc7, 0x1d, 0x5a, 0x57, 0x68, 0xba, 0xce, 0x4a, 0x55, 0xc5, 0x3a, 0x1d, 0xb4, 0xce, 0xf7, + 0x18, 0x8b, 0xc8, 0x4a, 0xa2, 0x99, 0xc7, 0xf2, 0x07, 0x46, 0x05, 0x6e, 0x35, 0xb7, 0xa0, 0xa8, + 0x89, 0x75, 0x32, 0xe8, 0x3a, 0x4c, 0xb7, 0x42, 0x2f, 0x60, 0x0b, 0x5b, 0xbd, 0x99, 0xcc, 0x98, + 0x79, 0x24, 0xaa, 0x69, 0x04, 0xdc, 0x59, 0x87, 0xca, 0xb4, 0xb2, 0x70, 0xe6, 0x1c, 0x4f, 0x4a, + 0xc6, 0xf9, 0x7c, 0x5e, 0x86, 0x15, 0x14, 0xad, 0xb2, 0x73, 0x99, 0x4b, 0x9f, 0x33, 0xb3, 0xf9, + 0xc1, 0x25, 0x74, 0x29, 0x95, 0xb3, 0x67, 0xea, 0x2f, 0x4e, 0x28, 0xd0, 0x7b, 0x23, 0xda, 0x72, + 0x42, 0x52, 0x0d, 0x83, 0x3a, 0x89, 0xb4, 0x20, 0xd0, 0x8f, 0xf3, 0xc0, 0x91, 0xf4, 0xde, 0xa8, + 0x65, 0x21, 0xe0, 0xec, 0x7a, 0xc8, 0xd5, 0x72, 0x71, 0x53, 0xae, 0x37, 0x9a, 0x39, 0xdf, 0xc5, + 0xbe, 0x29, 0xc5, 0x22, 0x27, 0x6b, 0xd1, 0x28, 0x8e, 0x70, 0x8a, 0x26, 0xfa, 0x36, 0x98, 0x12, + 0x71, 0x96, 0x92, 0x71, 0xbf, 0x90, 0x18, 0x4e, 0xe2, 0x14, 0x0c, 0x77, 0x60, 0xf3, 0xd0, 0xd7, + 0xce, 0x46, 0x93, 0x88, 0x45, 0x78, 0xcb, 0xf3, 0xb7, 0xa3, 0x99, 0x8b, 0xec, 0xab, 0x45, 0xe8, + 0xeb, 0x34, 0x14, 0x67, 0xd4, 0x98, 0xfd, 0x56, 0x98, 0xee, 0xb8, 0xb9, 0x8e, 0x14, 0x2e, 0xfe, + 0x4f, 0x06, 0xa1, 0xa4, 0xde, 0x00, 0xd0, 0x55, 0xf3, 0x69, 0xe7, 0x5c, 0xfa, 0x69, 0x67, 0x84, + 0x8a, 0x22, 0xfa, 0x6b, 0xce, 0xba, 0x61, 0x17, 0x58, 0xc8, 0x4f, 0xce, 0xa6, 0x0b, 0x13, 0x3d, + 0x7d, 0x0c, 0x35, 0x95, 0x4e, 0xb1, 0xef, 0x37, 0xa2, 0x81, 0xae, 0x5a, 0xa2, 0x3e, 0x73, 0x23, + 0xa3, 0x27, 0xa9, 0x3c, 0xe6, 0xae, 0x54, 0xd3, 0xc9, 0x42, 0xab, 0xb4, 0x10, 0x73, 0x18, 0x93, + 0x5b, 0x29, 0x9b, 0xc5, 0xe4, 0xd6, 0xe1, 0x87, 0x94, 0x5b, 0x25, 0x01, 0x9c, 0xd0, 0x42, 0x4d, + 0x98, 0xae, 0x9b, 0x79, 0x5e, 0x95, 0x5f, 0xe1, 0x93, 0x3d, 0x33, 0xae, 0xb6, 0xb5, 0xa4, 0x7a, + 0x8b, 0x69, 0x2a, 0xb8, 0x93, 0x30, 0x7a, 0x15, 0x46, 0xde, 0x0b, 0x22, 0xb6, 0x28, 0x05, 0xaf, + 0x21, 0xfd, 0xaf, 0x46, 0xde, 0xbc, 0x5d, 0x63, 0xe5, 0x87, 0xfb, 0xe5, 0xd1, 0x6a, 0xe0, 0xca, + 0xbf, 0x58, 0x55, 0x40, 0x0f, 0xe0, 0x8c, 0x71, 0x42, 0xab, 0xee, 0x42, 0xff, 0xdd, 0xbd, 0x20, + 0x9a, 0x3b, 0xb3, 0x92, 0x45, 0x09, 0x67, 0x37, 0x40, 0x8f, 0x3d, 0x3f, 0x10, 0x39, 0x92, 0x25, + 0x3f, 0xc3, 0xd8, 0x96, 0x92, 0xee, 0x7d, 0x9f, 0x42, 0xc0, 0x9d, 0x75, 0xec, 0x5f, 0xe6, 0x4f, + 0x26, 0x42, 0xb1, 0x4a, 0xa2, 0x76, 0xf3, 0x24, 0x52, 0x70, 0x2d, 0x19, 0x3a, 0xdf, 0x87, 0x7e, + 0x96, 0xfb, 0x35, 0x8b, 0x3d, 0xcb, 0xad, 0x93, 0x9d, 0x56, 0x93, 0x8a, 0xf7, 0x8f, 0xbe, 0xe3, + 0x6f, 0xc2, 0x48, 0x2c, 0x5a, 0xeb, 0x96, 0x35, 0x4c, 0xeb, 0x14, 0x7b, 0x9a, 0x54, 0x9c, 0x8e, + 0x2c, 0xc5, 0x8a, 0x8c, 0xfd, 0xcf, 0xf9, 0x0c, 0x48, 0xc8, 0x09, 0xe8, 0xdf, 0x2a, 0xa6, 0xfe, + 0xad, 0xdc, 0xe3, 0x0b, 0x72, 0xf4, 0x70, 0xff, 0xcc, 0xec, 0x37, 0x13, 0x2a, 0x3f, 0xea, 0xef, 0xc1, 0xf6, 0x0f, 0x59, 0x70, 0x3a, 0xcb, 0x80, 0x8a, 0x72, 0xa7, 0x5c, 0xa4, 0x55, 0xef, 0xe3, - 0x6a, 0x04, 0xef, 0x88, 0x72, 0xac, 0x30, 0x06, 0x4e, 0xc8, 0x71, 0xb4, 0x00, 0x75, 0xb7, 0x60, - 0xa2, 0x1e, 0x12, 0xed, 0x0e, 0x78, 0x9d, 0x3b, 0xf2, 0xf1, 0xfe, 0x3c, 0x77, 0x64, 0x27, 0x3e, - 0xfb, 0xa7, 0x0b, 0x70, 0x9a, 0x3f, 0x70, 0x2d, 0xec, 0x05, 0x9e, 0x5b, 0x0f, 0x5c, 0x91, 0x4c, - 0xe5, 0x2d, 0x18, 0x6f, 0x6b, 0x7a, 0x88, 0x5e, 0x21, 0xb2, 0x74, 0x7d, 0x45, 0x22, 0x0f, 0xea, - 0xa5, 0xd8, 0xa0, 0x85, 0x5c, 0x18, 0x27, 0x7b, 0x5e, 0x53, 0xbd, 0x92, 0x14, 0x8e, 0x7c, 0x37, - 0xa8, 0x56, 0x96, 0x35, 0x3a, 0xd8, 0xa0, 0xfa, 0x08, 0xf2, 0xeb, 0xd9, 0x3f, 0x6c, 0xc1, 0xe3, - 0x39, 0x01, 0xb5, 0x68, 0x73, 0xf7, 0xd8, 0x53, 0xa2, 0x48, 0xd5, 0xa5, 0x9a, 0xe3, 0x0f, 0x8c, - 0x58, 0x40, 0xd1, 0x67, 0x01, 0xf8, 0x03, 0x21, 0x15, 0x8f, 0xfa, 0x45, 0x1e, 0x32, 0x82, 0xa6, - 0x68, 0xc1, 0x2e, 0x64, 0x7d, 0xac, 0xd1, 0xb2, 0x7f, 0xa2, 0x08, 0xc3, 0xec, 0x41, 0x0a, 0xad, - 0xc0, 0xe8, 0x36, 0x0f, 0x31, 0x3d, 0x48, 0x34, 0xeb, 0x44, 0xce, 0xe4, 0x05, 0x58, 0x56, 0x46, - 0xab, 0x70, 0x8a, 0x87, 0xe8, 0x6e, 0x55, 0x49, 0xcb, 0xd9, 0x97, 0xea, 0x0a, 0x9e, 0xde, 0x4a, - 0x05, 0xee, 0xa8, 0x75, 0xa3, 0xe0, 0xac, 0x7a, 0xe8, 0x75, 0x98, 0xa4, 0xfc, 0x5d, 0xd0, 0x89, - 0x25, 0x25, 0x1e, 0x9c, 0x5b, 0x31, 0x94, 0xeb, 0x06, 0x14, 0xa7, 0xb0, 0xa9, 0xe0, 0xd5, 0xee, - 0x52, 0xcc, 0x0c, 0x27, 0x82, 0x97, 0xa9, 0x8c, 0x31, 0x71, 0x99, 0xe5, 0x54, 0x87, 0xd9, 0x89, - 0xad, 0x6f, 0x87, 0x24, 0xda, 0x0e, 0x5a, 0xae, 0xc8, 0x8e, 0x9e, 0x58, 0x4e, 0xa5, 0xe0, 0xb8, - 0xab, 0x06, 0xa5, 0xb2, 0xe9, 0x78, 0xad, 0x4e, 0x48, 0x12, 0x2a, 0x23, 0x26, 0x95, 0x95, 0x14, - 0x1c, 0x77, 0xd5, 0xa0, 0xeb, 0xe8, 0x8c, 0x48, 0x57, 0x2e, 0xc3, 0x09, 0x28, 0x73, 0xb8, 0x51, - 0xe9, 0x58, 0xd5, 0x23, 0x9e, 0x8e, 0x30, 0x18, 0x52, 0x09, 0xcf, 0x35, 0xf5, 0xa5, 0x70, 0xa9, - 0x92, 0x54, 0x1e, 0x26, 0x69, 0xf6, 0xf7, 0x17, 0xe0, 0x54, 0x86, 0xd9, 0x2d, 0x3f, 0xaa, 0xb6, + 0x6a, 0x04, 0xef, 0x8a, 0x72, 0xac, 0x30, 0xfa, 0x4e, 0xc8, 0x71, 0xb4, 0x00, 0x75, 0xb7, 0x61, + 0xbc, 0x1a, 0x12, 0xed, 0x0e, 0x78, 0x9d, 0x3b, 0xf2, 0xf1, 0xfe, 0x3c, 0x7b, 0x64, 0x27, 0x3e, + 0xfb, 0xa7, 0x0b, 0x70, 0x9a, 0x3f, 0x70, 0xcd, 0xef, 0x06, 0x9e, 0x5b, 0x0d, 0x5c, 0x91, 0x4c, + 0xe5, 0x2d, 0x18, 0x6b, 0x69, 0x7a, 0x88, 0x6e, 0x21, 0xb2, 0x74, 0x7d, 0x45, 0x22, 0x0f, 0xea, + 0xa5, 0xd8, 0xa0, 0x85, 0x5c, 0x18, 0x23, 0xbb, 0x5e, 0x5d, 0xbd, 0x92, 0x14, 0x8e, 0x7c, 0x37, + 0xa8, 0x56, 0x96, 0x34, 0x3a, 0xd8, 0xa0, 0xfa, 0x08, 0xf2, 0xeb, 0xd9, 0x3f, 0x6c, 0xc1, 0x63, + 0x39, 0x01, 0xb5, 0x68, 0x73, 0xf7, 0xd9, 0x53, 0xa2, 0x48, 0xd5, 0xa5, 0x9a, 0xe3, 0x0f, 0x8c, + 0x58, 0x40, 0xd1, 0xe7, 0x00, 0xf8, 0x03, 0x21, 0x15, 0x8f, 0x7a, 0x45, 0x1e, 0x32, 0x82, 0xa6, + 0x68, 0xc1, 0x2e, 0x64, 0x7d, 0xac, 0xd1, 0xb2, 0x7f, 0xa2, 0x08, 0x83, 0xec, 0x41, 0x0a, 0x2d, + 0xc3, 0xf0, 0x16, 0x0f, 0x31, 0xdd, 0x4f, 0x34, 0xeb, 0x44, 0xce, 0xe4, 0x05, 0x58, 0x56, 0x46, + 0xab, 0x70, 0x8a, 0x87, 0xe8, 0x6e, 0x56, 0x48, 0xd3, 0xd9, 0x93, 0xea, 0x0a, 0x9e, 0xde, 0x4a, + 0x05, 0xee, 0x58, 0xe9, 0x44, 0xc1, 0x59, 0xf5, 0xd0, 0xeb, 0x30, 0x41, 0xf9, 0xbb, 0xa0, 0x1d, + 0x4b, 0x4a, 0x3c, 0x38, 0xb7, 0x62, 0x28, 0xd7, 0x0d, 0x28, 0x4e, 0x61, 0x53, 0xc1, 0xab, 0xd5, + 0xa1, 0x98, 0x19, 0x4c, 0x04, 0x2f, 0x53, 0x19, 0x63, 0xe2, 0x32, 0xcb, 0xa9, 0x36, 0xb3, 0x13, + 0x5b, 0xdf, 0x0a, 0x49, 0xb4, 0x15, 0x34, 0x5d, 0x91, 0x1d, 0x3d, 0xb1, 0x9c, 0x4a, 0xc1, 0x71, + 0x47, 0x0d, 0x4a, 0x65, 0xd3, 0xf1, 0x9a, 0xed, 0x90, 0x24, 0x54, 0x86, 0x4c, 0x2a, 0xcb, 0x29, + 0x38, 0xee, 0xa8, 0x41, 0xd7, 0xd1, 0x19, 0x91, 0xae, 0x5c, 0x86, 0x13, 0x50, 0xe6, 0x70, 0xc3, + 0xd2, 0xb1, 0xaa, 0x4b, 0x3c, 0x1d, 0x61, 0x30, 0xa4, 0x12, 0x9e, 0x6b, 0xea, 0x4b, 0xe1, 0x52, + 0x25, 0xa9, 0x3c, 0x4c, 0xd2, 0xec, 0xef, 0x2f, 0xc0, 0xa9, 0x0c, 0xb3, 0x5b, 0x7e, 0x54, 0x35, 0xbc, 0x28, 0x56, 0x29, 0x7c, 0xb4, 0xa3, 0x8a, 0x97, 0x63, 0x85, 0x41, 0xf7, 0x03, 0x3f, 0x0c, - 0xd3, 0x07, 0xa0, 0x30, 0x6b, 0x13, 0xd0, 0x23, 0x26, 0xc3, 0xb9, 0x08, 0x43, 0x9d, 0x88, 0xc8, - 0x48, 0x58, 0xea, 0xfc, 0x66, 0x0a, 0x6d, 0x06, 0xa1, 0xac, 0xe9, 0x96, 0xd2, 0x25, 0x6b, 0xac, - 0x29, 0x57, 0x10, 0x73, 0x18, 0xed, 0x5c, 0x4c, 0x7c, 0xc7, 0x8f, 0x05, 0x03, 0x9b, 0xc4, 0x6f, - 0x61, 0xa5, 0x58, 0x40, 0xed, 0xaf, 0x14, 0xe1, 0x6c, 0xae, 0x21, 0x3e, 0xed, 0xfa, 0x6e, 0xe0, - 0x7b, 0x71, 0xa0, 0x1e, 0x45, 0x79, 0xcc, 0x16, 0xd2, 0xde, 0x5e, 0x15, 0xe5, 0x58, 0x61, 0xa0, - 0x4b, 0x32, 0xc1, 0x7e, 0x3a, 0x99, 0xd1, 0x62, 0xd5, 0xc8, 0xb1, 0x3f, 0x68, 0xa2, 0xb8, 0xa7, - 0x60, 0xa8, 0x1d, 0x04, 0xad, 0xf4, 0xa1, 0x45, 0xbb, 0x1b, 0x04, 0x2d, 0xcc, 0x80, 0xe8, 0x63, - 0x62, 0xbc, 0x52, 0xaf, 0x80, 0xd8, 0x71, 0x83, 0x48, 0x1b, 0xb4, 0x67, 0x60, 0x74, 0x87, 0xec, - 0x87, 0x9e, 0xbf, 0x95, 0x7e, 0x1d, 0xbe, 0xc1, 0x8b, 0xb1, 0x84, 0x9b, 0xa9, 0x2d, 0x46, 0x8f, - 0x3b, 0xc3, 0x5b, 0xa9, 0xef, 0x15, 0xf8, 0x03, 0x45, 0x98, 0xc2, 0x8b, 0xd5, 0x6f, 0x4e, 0xc4, - 0xed, 0xee, 0x89, 0x38, 0xee, 0x0c, 0x6f, 0xfd, 0x67, 0xe3, 0x17, 0x2c, 0x98, 0x62, 0xe1, 0x9f, - 0x45, 0xa4, 0x10, 0x2f, 0xf0, 0x4f, 0x80, 0xc5, 0x7b, 0x0a, 0x86, 0x43, 0xda, 0x68, 0x3a, 0x8b, - 0x11, 0xeb, 0x09, 0xe6, 0x30, 0x74, 0x0e, 0x86, 0x58, 0x17, 0xe8, 0xe4, 0x8d, 0xf3, 0x04, 0x10, - 0x55, 0x27, 0x76, 0x30, 0x2b, 0x65, 0xee, 0xef, 0x98, 0xb4, 0x5b, 0x1e, 0xef, 0x74, 0xf2, 0x04, - 0xf2, 0xe1, 0x70, 0x7f, 0xcf, 0xec, 0xda, 0xfb, 0x73, 0x7f, 0xcf, 0x26, 0xd9, 0x5b, 0x7c, 0xfa, - 0xc3, 0x02, 0x5c, 0xc8, 0xac, 0x37, 0xb0, 0xfb, 0x7b, 0xef, 0xda, 0xc7, 0x63, 0xe4, 0x93, 0x6d, - 0x7b, 0x53, 0x3c, 0x41, 0xdb, 0x9b, 0xa1, 0x41, 0x39, 0xcc, 0xe1, 0x01, 0xbc, 0xd2, 0x33, 0x87, - 0xec, 0x43, 0xe2, 0x95, 0x9e, 0xd9, 0xb7, 0x1c, 0xf1, 0xef, 0xcf, 0x0a, 0x39, 0xdf, 0xc2, 0x04, - 0xc1, 0xcb, 0xf4, 0x9c, 0x61, 0xc0, 0x48, 0x70, 0xcc, 0xe3, 0xfc, 0x8c, 0xe1, 0x65, 0x58, 0x41, - 0x91, 0xa7, 0xf9, 0x77, 0x17, 0xf2, 0x93, 0x7a, 0xe6, 0x36, 0x35, 0x6f, 0xbe, 0x58, 0xa9, 0x21, - 0xc8, 0xf0, 0xf5, 0x5e, 0xd5, 0x84, 0xf7, 0xe2, 0xe0, 0xc2, 0xfb, 0x78, 0xb6, 0xe0, 0x8e, 0x16, - 0x60, 0x6a, 0xd7, 0xf3, 0xe9, 0xb1, 0xb9, 0x6f, 0xb2, 0xac, 0x2a, 0xdc, 0xc9, 0xaa, 0x09, 0xc6, - 0x69, 0xfc, 0xb9, 0x57, 0x61, 0xe2, 0xe1, 0xd5, 0x96, 0xdf, 0x28, 0xc2, 0x13, 0x3d, 0xb6, 0x3d, - 0x3f, 0xeb, 0x8d, 0x39, 0xd0, 0xce, 0xfa, 0xae, 0x79, 0xa8, 0xc3, 0xe9, 0xcd, 0x4e, 0xab, 0xb5, - 0xcf, 0xcc, 0x5b, 0x89, 0x2b, 0x31, 0x04, 0x4f, 0x79, 0x4e, 0xa6, 0xdc, 0x58, 0xc9, 0xc0, 0xc1, - 0x99, 0x35, 0xd1, 0x1b, 0x80, 0x02, 0x91, 0x51, 0xf8, 0x1a, 0xf1, 0xc5, 0x3b, 0x00, 0x1b, 0xf8, - 0x62, 0xb2, 0x19, 0x6f, 0x75, 0x61, 0xe0, 0x8c, 0x5a, 0x54, 0x38, 0xa0, 0xb7, 0xd2, 0xbe, 0xea, - 0x56, 0x4a, 0x38, 0xc0, 0x3a, 0x10, 0x9b, 0xb8, 0xe8, 0x1a, 0xcc, 0x38, 0x7b, 0x8e, 0xc7, 0xc3, - 0x00, 0x4a, 0x02, 0x5c, 0x3a, 0x50, 0xca, 0xb2, 0x85, 0x34, 0x02, 0xee, 0xae, 0x93, 0xf2, 0x00, - 0x1f, 0xc9, 0xf7, 0x00, 0xef, 0x7d, 0x2e, 0xf6, 0xd3, 0xfd, 0xda, 0xff, 0xd5, 0xa2, 0xd7, 0x97, - 0x96, 0xe8, 0x5f, 0x3d, 0xb6, 0xbe, 0xca, 0x0c, 0x58, 0xb8, 0x32, 0x50, 0x73, 0xc6, 0x3e, 0xa3, - 0x19, 0xb0, 0x24, 0x40, 0x6c, 0xe2, 0xf2, 0x05, 0x11, 0x25, 0x3e, 0x40, 0x06, 0x8b, 0x2f, 0x82, - 0x39, 0x28, 0x0c, 0xf4, 0x39, 0x18, 0x75, 0xbd, 0x3d, 0x2f, 0x0a, 0x42, 0xb1, 0x59, 0x8e, 0xe8, - 0x49, 0x91, 0x9c, 0x83, 0x55, 0x4e, 0x06, 0x4b, 0x7a, 0xf6, 0x0f, 0x14, 0x60, 0x42, 0xb6, 0xf8, - 0x66, 0x27, 0x88, 0x9d, 0x13, 0xb8, 0x96, 0xaf, 0x19, 0xd7, 0xf2, 0xc7, 0x7a, 0x45, 0xb4, 0x60, - 0x5d, 0xca, 0xbd, 0x8e, 0x6f, 0xa5, 0xae, 0xe3, 0xa7, 0xfb, 0x93, 0xea, 0x7d, 0x0d, 0xff, 0x0b, - 0x0b, 0x66, 0x0c, 0xfc, 0x13, 0xb8, 0x0d, 0x56, 0xcc, 0xdb, 0xe0, 0xc9, 0xbe, 0xdf, 0x90, 0x73, - 0x0b, 0x7c, 0x6f, 0x31, 0xd5, 0x77, 0x76, 0xfa, 0xbf, 0x0b, 0x43, 0xdb, 0x4e, 0xe8, 0xf6, 0x8a, - 0x9c, 0xdb, 0x55, 0x69, 0xfe, 0xba, 0x13, 0xba, 0xfc, 0x0c, 0x7f, 0x4e, 0xa5, 0xe5, 0x74, 0x42, - 0xb7, 0xaf, 0xcb, 0x1b, 0x6b, 0x0a, 0xbd, 0x02, 0x23, 0x51, 0x33, 0x68, 0x2b, 0x83, 0xd4, 0x8b, - 0x3c, 0x65, 0x27, 0x2d, 0x39, 0x3c, 0xa8, 0x20, 0xb3, 0x39, 0x5a, 0x8c, 0x05, 0x3e, 0x7a, 0x0b, - 0x26, 0xd8, 0x2f, 0x65, 0x29, 0x51, 0xcc, 0xcf, 0xd7, 0xd0, 0xd0, 0x11, 0xb9, 0xc1, 0x8d, 0x51, - 0x84, 0x4d, 0x52, 0x73, 0x5b, 0x50, 0x56, 0x9f, 0xf5, 0x48, 0x5d, 0x95, 0xfe, 0x53, 0x11, 0x4e, - 0x65, 0xac, 0x39, 0x14, 0x19, 0x33, 0xf1, 0xc2, 0x80, 0x4b, 0xf5, 0x7d, 0xce, 0x45, 0xc4, 0xa4, - 0x21, 0x57, 0xac, 0xad, 0x81, 0x1b, 0xbd, 0x1d, 0x91, 0x74, 0xa3, 0xb4, 0xa8, 0x7f, 0xa3, 0xb4, - 0xb1, 0x13, 0x1b, 0x6a, 0xda, 0x90, 0xea, 0xe9, 0x23, 0x9d, 0xd3, 0x3f, 0x2e, 0xc2, 0xe9, 0xac, - 0x20, 0x3b, 0xe8, 0x3b, 0x53, 0xb9, 0x7b, 0x5e, 0x1a, 0x34, 0x3c, 0x0f, 0x4f, 0xe8, 0x23, 0x52, - 0x6f, 0xcf, 0x9b, 0xd9, 0x7c, 0xfa, 0x0e, 0xb3, 0x68, 0x93, 0xf9, 0xb7, 0x86, 0x3c, 0xe7, 0x92, - 0x3c, 0x3e, 0x3e, 0x39, 0x70, 0x07, 0x44, 0xb2, 0xa6, 0x28, 0xe5, 0xdf, 0x2a, 0x8b, 0xfb, 0xfb, - 0xb7, 0xca, 0x96, 0xe7, 0x3c, 0x18, 0xd3, 0xbe, 0xe6, 0x91, 0xce, 0xf8, 0x0e, 0xbd, 0xad, 0xb4, - 0x7e, 0x3f, 0xd2, 0x59, 0xff, 0x61, 0x0b, 0x52, 0xd6, 0x9f, 0x4a, 0x2d, 0x66, 0xe5, 0xaa, 0xc5, - 0x2e, 0xc2, 0x50, 0x18, 0xb4, 0x48, 0x3a, 0x55, 0x0e, 0x0e, 0x5a, 0x04, 0x33, 0x08, 0xc5, 0x88, - 0x13, 0x65, 0xc7, 0xb8, 0x2e, 0xc8, 0x09, 0x11, 0xed, 0x29, 0x18, 0x6e, 0x91, 0x3d, 0xd2, 0x4a, - 0xc7, 0xa1, 0xbf, 0x49, 0x0b, 0x31, 0x87, 0xd9, 0xbf, 0x30, 0x04, 0xe7, 0x7b, 0x7a, 0x88, 0x53, - 0x71, 0x68, 0xcb, 0x89, 0xc9, 0x3d, 0x67, 0x3f, 0x1d, 0x30, 0xfa, 0x1a, 0x2f, 0xc6, 0x12, 0xce, - 0x0c, 0xe2, 0x79, 0x80, 0xc8, 0x94, 0x12, 0x51, 0xc4, 0x85, 0x14, 0x50, 0x53, 0x29, 0x55, 0x3c, - 0x0e, 0xa5, 0xd4, 0x55, 0x80, 0x28, 0x6a, 0x71, 0xfb, 0x02, 0x57, 0x58, 0xda, 0x27, 0x81, 0x44, - 0x1b, 0x37, 0x05, 0x04, 0x6b, 0x58, 0xa8, 0x0a, 0xd3, 0xed, 0x30, 0x88, 0xb9, 0x4e, 0xb6, 0xca, - 0x0d, 0x93, 0x86, 0x4d, 0xe7, 0xdc, 0x7a, 0x0a, 0x8e, 0xbb, 0x6a, 0xa0, 0x97, 0x61, 0x4c, 0x38, - 0xec, 0xd6, 0x83, 0xa0, 0x25, 0xd4, 0x40, 0xca, 0xcc, 0xa5, 0x91, 0x80, 0xb0, 0x8e, 0xa7, 0x55, - 0x63, 0x8a, 0xde, 0xd1, 0xcc, 0x6a, 0x5c, 0xd9, 0xab, 0xe1, 0xa5, 0x02, 0x6e, 0x95, 0x06, 0x0a, - 0xb8, 0x95, 0x28, 0xc6, 0xca, 0x03, 0xbf, 0x6d, 0x41, 0x5f, 0x55, 0xd2, 0xcf, 0x0e, 0xc1, 0x29, - 0xb1, 0x70, 0x1e, 0xf5, 0x72, 0xb9, 0xdd, 0xbd, 0x5c, 0x8e, 0x43, 0x75, 0xf6, 0xcd, 0x35, 0x73, - 0xd2, 0x6b, 0xe6, 0x07, 0x2d, 0x30, 0xd9, 0x2b, 0xf4, 0x97, 0x72, 0x23, 0xee, 0xbf, 0x9c, 0xcb, - 0xae, 0xb9, 0xf2, 0x02, 0x79, 0x9f, 0xb1, 0xf7, 0xed, 0xff, 0x62, 0xc1, 0x93, 0x7d, 0x29, 0xa2, - 0x65, 0x28, 0x33, 0x1e, 0x50, 0x93, 0xce, 0x9e, 0x56, 0x86, 0x8b, 0x12, 0x90, 0xc3, 0x92, 0x26, - 0x35, 0xd1, 0x72, 0x57, 0x6a, 0x83, 0x67, 0x32, 0x52, 0x1b, 0x9c, 0x31, 0x86, 0xe7, 0x21, 0x73, - 0x1b, 0xfc, 0x72, 0x11, 0x46, 0xf8, 0x8a, 0x3f, 0x01, 0x31, 0x6c, 0x45, 0xe8, 0x6d, 0x7b, 0x84, - 0xdc, 0xe2, 0x7d, 0x99, 0xaf, 0x3a, 0xb1, 0xc3, 0xd9, 0x04, 0x75, 0x5b, 0x25, 0x1a, 0x5e, 0x34, - 0x6f, 0xdc, 0x67, 0x73, 0x29, 0xc5, 0x24, 0x70, 0x1a, 0xda, 0xed, 0xf6, 0x05, 0x80, 0x28, 0x0e, - 0x3d, 0x7f, 0x8b, 0xd2, 0x10, 0xc1, 0xdb, 0x3e, 0xde, 0xa3, 0xf5, 0x86, 0x42, 0xe6, 0x7d, 0x48, - 0x76, 0xba, 0x02, 0x60, 0x8d, 0xe2, 0xdc, 0xa7, 0xa0, 0xac, 0x90, 0xfb, 0x69, 0x71, 0xc6, 0x75, - 0xe6, 0xe2, 0x33, 0x30, 0x95, 0x6a, 0xeb, 0x48, 0x4a, 0xa0, 0x5f, 0xb4, 0x60, 0x8a, 0x77, 0x79, - 0xd9, 0xdf, 0x13, 0x67, 0xea, 0x7b, 0x70, 0xba, 0x95, 0x71, 0xb6, 0x89, 0x19, 0x1d, 0xfc, 0x2c, - 0x54, 0x4a, 0x9f, 0x2c, 0x28, 0xce, 0x6c, 0x03, 0x5d, 0xa6, 0xeb, 0x96, 0x9e, 0x5d, 0x4e, 0x4b, - 0x38, 0x57, 0x8d, 0xf3, 0x35, 0xcb, 0xcb, 0xb0, 0x82, 0xda, 0xbf, 0x65, 0xc1, 0x0c, 0xef, 0xf9, - 0x0d, 0xb2, 0xaf, 0x76, 0xf8, 0x07, 0xd9, 0x77, 0x91, 0x6d, 0xa4, 0x90, 0x93, 0x6d, 0x44, 0xff, - 0xb4, 0x62, 0xcf, 0x4f, 0xfb, 0x69, 0x0b, 0xc4, 0x0a, 0x3c, 0x01, 0x51, 0xfe, 0x5b, 0x4d, 0x51, - 0x7e, 0x2e, 0x7f, 0x51, 0xe7, 0xc8, 0xf0, 0x7f, 0x6a, 0xc1, 0x34, 0x47, 0x48, 0xde, 0x9c, 0x3f, - 0xd0, 0x79, 0x18, 0x24, 0x6d, 0xa0, 0xca, 0x25, 0x9e, 0xfd, 0x51, 0xc6, 0x64, 0x0d, 0xf5, 0x9c, - 0x2c, 0x57, 0x6e, 0xa0, 0x23, 0xa4, 0xcc, 0x3c, 0x72, 0xd4, 0x6e, 0xfb, 0x0f, 0x2c, 0x40, 0xbc, - 0x19, 0x83, 0xfd, 0xa1, 0x4c, 0x05, 0x2b, 0xd5, 0xae, 0x8b, 0xe4, 0xa8, 0x51, 0x10, 0xac, 0x61, - 0x1d, 0xcb, 0xf0, 0xa4, 0x0c, 0x07, 0x8a, 0xfd, 0x0d, 0x07, 0x8e, 0x30, 0xa2, 0xbf, 0x3f, 0x0c, - 0x69, 0xf7, 0x03, 0x74, 0x07, 0xc6, 0x9b, 0x4e, 0xdb, 0xd9, 0xf0, 0x5a, 0x5e, 0xec, 0x91, 0xa8, - 0x97, 0xc5, 0xd1, 0x92, 0x86, 0x27, 0x9e, 0x7a, 0xb5, 0x12, 0x6c, 0xd0, 0x41, 0xf3, 0x00, 0xed, - 0xd0, 0xdb, 0xf3, 0x5a, 0x64, 0x8b, 0x69, 0x1c, 0x98, 0x3b, 0x27, 0x37, 0xa3, 0x91, 0xa5, 0x58, - 0xc3, 0xc8, 0xf0, 0xcc, 0x2b, 0x3e, 0x3a, 0xcf, 0xbc, 0xa1, 0x23, 0x7a, 0xe6, 0x0d, 0x0f, 0xe4, - 0x99, 0x87, 0xe1, 0x31, 0xc9, 0x22, 0xd1, 0xff, 0x2b, 0x5e, 0x8b, 0x08, 0xbe, 0x98, 0x3b, 0x79, - 0xce, 0x3d, 0x38, 0xa8, 0x3c, 0x86, 0x33, 0x31, 0x70, 0x4e, 0x4d, 0xf4, 0x59, 0x98, 0x75, 0x5a, - 0xad, 0xe0, 0x9e, 0x1a, 0xb5, 0xe5, 0xa8, 0xe9, 0xb4, 0xb8, 0xc6, 0x7e, 0x94, 0x51, 0x3d, 0xf7, - 0xe0, 0xa0, 0x32, 0xbb, 0x90, 0x83, 0x83, 0x73, 0x6b, 0xa7, 0x1c, 0xfb, 0x4a, 0x7d, 0x1d, 0xfb, - 0x5e, 0x83, 0x72, 0x3b, 0x0c, 0x9a, 0xab, 0x9a, 0xf7, 0xcf, 0x05, 0x96, 0x90, 0x5f, 0x16, 0x1e, - 0x1e, 0x54, 0x26, 0xd4, 0x1f, 0x76, 0xc3, 0x27, 0x15, 0x32, 0xfc, 0xf9, 0xe0, 0x51, 0xfa, 0xf3, - 0xed, 0xc0, 0xa9, 0x06, 0x09, 0x3d, 0x96, 0x59, 0xd4, 0x4d, 0xce, 0x8f, 0x75, 0x28, 0x87, 0xa9, - 0x13, 0x73, 0xa0, 0x30, 0x55, 0x5a, 0xf4, 0x64, 0x79, 0x42, 0x26, 0x84, 0xec, 0x3f, 0xb1, 0x60, - 0x54, 0x18, 0xbe, 0x9f, 0x00, 0xa3, 0xb6, 0x60, 0xe8, 0xcb, 0x2b, 0xd9, 0xb7, 0x0a, 0xeb, 0x4c, - 0xae, 0xa6, 0xbc, 0x96, 0xd2, 0x94, 0x3f, 0xd9, 0x8b, 0x48, 0x6f, 0x1d, 0xf9, 0xdf, 0x2d, 0xc2, - 0xa4, 0xe9, 0xab, 0x72, 0x02, 0x43, 0xb0, 0x06, 0xa3, 0x91, 0x70, 0x8c, 0x2a, 0xe4, 0x1b, 0x74, - 0xa7, 0x27, 0x31, 0xb1, 0xd6, 0x12, 0xae, 0x50, 0x92, 0x48, 0xa6, 0xc7, 0x55, 0xf1, 0x11, 0x7a, - 0x5c, 0xf5, 0x73, 0x17, 0x1a, 0x3a, 0x0e, 0x77, 0x21, 0xfb, 0x6b, 0xec, 0x66, 0xd3, 0xcb, 0x4f, - 0x80, 0xe9, 0xb9, 0x66, 0xde, 0x81, 0x76, 0x8f, 0x95, 0x25, 0x3a, 0x95, 0xc3, 0xfc, 0xfc, 0xbc, - 0x05, 0xe7, 0x33, 0xbe, 0x4a, 0xe3, 0x84, 0x9e, 0x83, 0x92, 0xd3, 0x71, 0x3d, 0xb5, 0x97, 0xb5, - 0x57, 0xb3, 0x05, 0x51, 0x8e, 0x15, 0x06, 0x5a, 0x82, 0x19, 0x72, 0xbf, 0xed, 0xf1, 0x67, 0x4b, - 0xdd, 0xa4, 0xb2, 0xc8, 0x43, 0xf7, 0x2e, 0xa7, 0x81, 0xb8, 0x1b, 0x5f, 0x39, 0xb7, 0x17, 0x73, - 0x9d, 0xdb, 0xff, 0xb1, 0x05, 0x63, 0xca, 0x09, 0xe6, 0x91, 0x8f, 0xf6, 0xb7, 0x99, 0xa3, 0xfd, - 0x44, 0x8f, 0xd1, 0xce, 0x19, 0xe6, 0xbf, 0x5f, 0x50, 0xfd, 0xad, 0x07, 0x61, 0x3c, 0x00, 0x87, - 0xf5, 0x0a, 0x94, 0xda, 0x61, 0x10, 0x07, 0xcd, 0xa0, 0x25, 0x18, 0xac, 0x73, 0x49, 0xec, 0x05, - 0x5e, 0x7e, 0xa8, 0xfd, 0xc6, 0x0a, 0x9b, 0x8d, 0x5e, 0x10, 0xc6, 0x82, 0xa9, 0x49, 0x46, 0x2f, - 0x08, 0x63, 0xcc, 0x20, 0xc8, 0x05, 0x88, 0x9d, 0x70, 0x8b, 0xc4, 0xb4, 0x4c, 0x84, 0x71, 0xc9, - 0x3f, 0x3c, 0x3a, 0xb1, 0xd7, 0x9a, 0xf7, 0xfc, 0x38, 0x8a, 0xc3, 0xf9, 0x9a, 0x1f, 0xdf, 0x0a, - 0xb9, 0xbc, 0xa6, 0x05, 0x53, 0x50, 0xb4, 0xb0, 0x46, 0x57, 0xba, 0xa0, 0xb2, 0x36, 0x86, 0xcd, - 0xf7, 0xf7, 0x35, 0x51, 0x8e, 0x15, 0x86, 0xfd, 0x29, 0x76, 0x95, 0xb0, 0x01, 0x3a, 0x5a, 0x9c, - 0x83, 0xaf, 0x97, 0xd4, 0xd0, 0xb2, 0xc7, 0xb7, 0xaa, 0x1e, 0x4d, 0xa1, 0xf7, 0xc9, 0x4d, 0x1b, - 0xd6, 0xdd, 0x7b, 0x92, 0x90, 0x0b, 0xe8, 0xdb, 0xbb, 0xcc, 0x32, 0x9e, 0xef, 0x73, 0x05, 0x1c, - 0xc1, 0x10, 0x83, 0x85, 0x13, 0x67, 0xc1, 0x96, 0x6b, 0x75, 0xb1, 0xc8, 0xb5, 0x70, 0xe2, 0x02, - 0x80, 0x13, 0x1c, 0x74, 0x45, 0x48, 0xfb, 0x43, 0x46, 0x52, 0x41, 0x29, 0xed, 0xcb, 0xcf, 0xd7, - 0xc4, 0xfd, 0x17, 0x60, 0x4c, 0x25, 0x17, 0xac, 0xf3, 0x1c, 0x6d, 0x22, 0xa8, 0xcd, 0x72, 0x52, - 0x8c, 0x75, 0x1c, 0xb4, 0x0e, 0x53, 0x11, 0x57, 0xf5, 0xa8, 0xd8, 0x85, 0x5c, 0x65, 0xf6, 0x71, - 0x69, 0xce, 0xd1, 0x30, 0xc1, 0x87, 0xac, 0x88, 0x1f, 0x1d, 0xd2, 0x8f, 0x34, 0x4d, 0x02, 0xbd, - 0x0e, 0x93, 0x2d, 0x3d, 0x8d, 0x7f, 0x5d, 0x68, 0xd4, 0x94, 0x55, 0xb4, 0x91, 0xe4, 0xbf, 0x8e, - 0x53, 0xd8, 0x94, 0x31, 0xd3, 0x4b, 0x44, 0xbc, 0x4d, 0xc7, 0xdf, 0x22, 0x91, 0x48, 0x8d, 0xc6, - 0x18, 0xb3, 0x9b, 0x39, 0x38, 0x38, 0xb7, 0x36, 0x7a, 0x05, 0xc6, 0xe5, 0xe7, 0x6b, 0x5e, 0xd2, - 0x89, 0xed, 0xbd, 0x06, 0xc3, 0x06, 0x26, 0xba, 0x07, 0x67, 0xe4, 0xff, 0xf5, 0xd0, 0xd9, 0xdc, - 0xf4, 0x9a, 0xc2, 0x49, 0x9d, 0x3b, 0x20, 0x2d, 0x48, 0x8f, 0xa6, 0xe5, 0x2c, 0xa4, 0xc3, 0x83, - 0xca, 0x45, 0x31, 0x6a, 0x99, 0x70, 0x36, 0x89, 0xd9, 0xf4, 0xd1, 0x2a, 0x9c, 0xda, 0x26, 0x4e, - 0x2b, 0xde, 0x5e, 0xda, 0x26, 0xcd, 0x1d, 0xb9, 0x89, 0x98, 0xef, 0xb5, 0x66, 0xb1, 0x7e, 0xbd, - 0x1b, 0x05, 0x67, 0xd5, 0x43, 0x6f, 0xc3, 0x6c, 0xbb, 0xb3, 0xd1, 0xf2, 0xa2, 0xed, 0xb5, 0x20, - 0x66, 0x16, 0x24, 0x2a, 0x37, 0x9f, 0x70, 0xd2, 0x56, 0x7e, 0xe7, 0xf5, 0x1c, 0x3c, 0x9c, 0x4b, - 0x01, 0xbd, 0x07, 0x67, 0x52, 0x8b, 0x41, 0xb8, 0x8c, 0x4e, 0xe6, 0x47, 0x2f, 0x6e, 0x64, 0x55, - 0x10, 0x2e, 0xa0, 0x59, 0x20, 0x9c, 0xdd, 0xc4, 0xfb, 0xb3, 0x2b, 0x7a, 0x97, 0x56, 0xd6, 0x98, - 0x32, 0xf4, 0x45, 0x18, 0xd7, 0x57, 0x91, 0xb8, 0x60, 0x2e, 0x65, 0xf3, 0x2c, 0xda, 0x6a, 0xe3, - 0x2c, 0x9d, 0x5a, 0x51, 0x3a, 0x0c, 0x1b, 0x14, 0x6d, 0x02, 0xd9, 0xdf, 0x87, 0x6e, 0x42, 0xa9, - 0xd9, 0xf2, 0x88, 0x1f, 0xd7, 0xea, 0xbd, 0x42, 0xa8, 0x2c, 0x09, 0x1c, 0x31, 0x60, 0x22, 0xdc, - 0x2b, 0x2f, 0xc3, 0x8a, 0x82, 0xfd, 0xab, 0x05, 0xa8, 0xf4, 0x89, 0x1d, 0x9c, 0x52, 0x7f, 0x5b, - 0x03, 0xa9, 0xbf, 0x17, 0x64, 0xa6, 0xc1, 0xb5, 0x94, 0x4e, 0x20, 0x95, 0x45, 0x30, 0xd1, 0x0c, - 0xa4, 0xf1, 0x07, 0x36, 0x47, 0xd6, 0x35, 0xe8, 0x43, 0x7d, 0x0d, 0xea, 0x8d, 0x97, 0xb3, 0xe1, - 0xc1, 0x05, 0x91, 0xdc, 0x57, 0x10, 0xfb, 0x6b, 0x05, 0x38, 0xa3, 0x86, 0xf0, 0x2f, 0xee, 0xc0, - 0xdd, 0xee, 0x1e, 0xb8, 0x63, 0x78, 0x43, 0xb2, 0x6f, 0xc1, 0x08, 0x0f, 0x41, 0x33, 0x00, 0x03, - 0xf4, 0x94, 0x19, 0xaf, 0x4c, 0x5d, 0xd3, 0x46, 0xcc, 0xb2, 0xbf, 0x66, 0xc1, 0xd4, 0xfa, 0x52, - 0xbd, 0x11, 0x34, 0x77, 0x48, 0xbc, 0xc0, 0x19, 0x56, 0x2c, 0xf8, 0x1f, 0xeb, 0x21, 0xf9, 0x9a, - 0x2c, 0x8e, 0xe9, 0x22, 0x0c, 0x6d, 0x07, 0x51, 0x9c, 0x7e, 0x60, 0xbe, 0x1e, 0x44, 0x31, 0x66, - 0x10, 0xfb, 0xb7, 0x2d, 0x18, 0x66, 0xf9, 0x71, 0xfb, 0x25, 0x6d, 0x1e, 0xe4, 0xbb, 0xd0, 0xcb, - 0x30, 0x42, 0x36, 0x37, 0x49, 0x33, 0x16, 0xb3, 0x2a, 0xbd, 0x64, 0x47, 0x96, 0x59, 0x29, 0xbd, - 0xf4, 0x59, 0x63, 0xfc, 0x2f, 0x16, 0xc8, 0xe8, 0x2e, 0x94, 0x63, 0x6f, 0x97, 0x2c, 0xb8, 0xae, - 0x78, 0xa2, 0x7b, 0x08, 0xa7, 0xe4, 0x75, 0x49, 0x00, 0x27, 0xb4, 0xec, 0xaf, 0x14, 0x00, 0x92, - 0xc8, 0x06, 0xfd, 0x3e, 0x71, 0xb1, 0xeb, 0xf1, 0xe6, 0x52, 0xc6, 0xe3, 0x0d, 0x4a, 0x08, 0x66, - 0xbc, 0xdc, 0xa8, 0x61, 0x2a, 0x0e, 0x34, 0x4c, 0x43, 0x47, 0x19, 0xa6, 0x25, 0x98, 0x49, 0x22, - 0x33, 0x98, 0x61, 0x6a, 0x98, 0x90, 0xb2, 0x9e, 0x06, 0xe2, 0x6e, 0x7c, 0x9b, 0xc0, 0x45, 0x19, - 0x9f, 0x54, 0xde, 0x35, 0xcc, 0x02, 0xf4, 0x08, 0xf9, 0xbb, 0x93, 0xd7, 0xa9, 0x42, 0xee, 0xeb, - 0xd4, 0x8f, 0x59, 0x70, 0x3a, 0xdd, 0x0e, 0x73, 0xc9, 0xfb, 0xb2, 0x05, 0x67, 0xd8, 0x1b, 0x1d, - 0x6b, 0xb5, 0xfb, 0x45, 0xf0, 0xa5, 0xec, 0x88, 0x15, 0xbd, 0x7b, 0x9c, 0xb8, 0x63, 0xaf, 0x66, - 0x91, 0xc6, 0xd9, 0x2d, 0xda, 0x5f, 0xb6, 0xe0, 0x6c, 0x6e, 0x5a, 0x26, 0x74, 0x19, 0x4a, 0x4e, - 0xdb, 0xe3, 0x0a, 0x30, 0xb1, 0xdf, 0x99, 0xf4, 0x58, 0xaf, 0x71, 0xf5, 0x97, 0x82, 0xaa, 0x74, - 0x91, 0x85, 0xdc, 0x74, 0x91, 0x7d, 0xb3, 0x3f, 0xda, 0xdf, 0x67, 0x81, 0xf0, 0xc2, 0x1a, 0xe0, - 0x90, 0x79, 0x4b, 0x66, 0xdb, 0x35, 0x42, 0xc3, 0x5f, 0xcc, 0x77, 0x4b, 0x13, 0x01, 0xe1, 0xd5, - 0xa5, 0x6e, 0x84, 0x81, 0x37, 0x68, 0xd9, 0x2e, 0x08, 0x68, 0x95, 0x30, 0x9d, 0x55, 0xff, 0xde, - 0x5c, 0x05, 0x70, 0x19, 0xae, 0x96, 0x73, 0x53, 0x5d, 0x21, 0x55, 0x05, 0xc1, 0x1a, 0x96, 0xfd, - 0x1f, 0x0a, 0x30, 0x26, 0x43, 0x91, 0x77, 0xfc, 0x41, 0x24, 0xcb, 0x23, 0xe5, 0x26, 0x62, 0x49, - 0x6a, 0x29, 0xe1, 0x7a, 0x22, 0x90, 0x27, 0x49, 0x6a, 0x25, 0x00, 0x27, 0x38, 0xe8, 0x19, 0x18, - 0x8d, 0x3a, 0x1b, 0x0c, 0x3d, 0xe5, 0x33, 0xd4, 0xe0, 0xc5, 0x58, 0xc2, 0xd1, 0x67, 0x61, 0x9a, - 0xd7, 0x0b, 0x83, 0xb6, 0xb3, 0xc5, 0xb5, 0xad, 0xc3, 0xca, 0xd9, 0x77, 0x7a, 0x35, 0x05, 0x3b, - 0x3c, 0xa8, 0x9c, 0x4e, 0x97, 0x31, 0x3d, 0x7d, 0x17, 0x15, 0xf6, 0xf6, 0xcf, 0x1b, 0xa1, 0xcb, - 0xb4, 0xcb, 0x64, 0x20, 0x01, 0x61, 0x1d, 0xcf, 0xfe, 0x22, 0xa0, 0xee, 0xa0, 0xec, 0xe8, 0x0d, - 0x6e, 0xf0, 0xe5, 0x85, 0xc4, 0xed, 0xa5, 0xb7, 0xd7, 0x5d, 0x5a, 0xa5, 0xb9, 0x3f, 0xaf, 0x85, - 0x55, 0x7d, 0xfb, 0x6f, 0x14, 0x61, 0x3a, 0xed, 0xe0, 0x88, 0xae, 0xc3, 0x08, 0xbf, 0x23, 0x05, - 0xf9, 0x1e, 0xcf, 0xc2, 0x9a, 0x5b, 0x24, 0x3b, 0x2d, 0xc4, 0x35, 0x2b, 0xea, 0xa3, 0xb7, 0x61, - 0xcc, 0x0d, 0xee, 0xf9, 0xf7, 0x9c, 0xd0, 0x5d, 0xa8, 0xd7, 0xc4, 0x72, 0xce, 0x64, 0xb5, 0xab, - 0x09, 0x9a, 0xee, 0x6a, 0xc9, 0x9e, 0x40, 0x12, 0x10, 0xd6, 0xc9, 0xa1, 0x75, 0x16, 0x68, 0x72, - 0xd3, 0xdb, 0x5a, 0x75, 0xda, 0xbd, 0xac, 0x7f, 0x97, 0x24, 0x92, 0x46, 0x79, 0x42, 0x44, 0xa3, - 0xe4, 0x00, 0x9c, 0x10, 0x42, 0xdf, 0x09, 0xa7, 0xa2, 0x1c, 0xed, 0x5c, 0x5e, 0x8e, 0x8e, 0x5e, - 0x0a, 0xab, 0xc5, 0xc7, 0xa9, 0x10, 0x94, 0xa5, 0xc7, 0xcb, 0x6a, 0xc6, 0xfe, 0xb5, 0x53, 0x60, - 0x6c, 0x62, 0x23, 0x65, 0x93, 0x75, 0x4c, 0x29, 0x9b, 0x30, 0x94, 0xc8, 0x6e, 0x3b, 0xde, 0xaf, - 0x7a, 0x61, 0xaf, 0x94, 0x82, 0xcb, 0x02, 0xa7, 0x9b, 0xa6, 0x84, 0x60, 0x45, 0x27, 0x3b, 0xaf, - 0x56, 0xf1, 0x03, 0xcc, 0xab, 0x35, 0x74, 0x82, 0x79, 0xb5, 0xd6, 0x60, 0x74, 0xcb, 0x8b, 0x31, - 0x69, 0x07, 0x82, 0x3b, 0xcd, 0x5c, 0x87, 0xd7, 0x38, 0x4a, 0x77, 0x06, 0x17, 0x01, 0xc0, 0x92, - 0x08, 0x7a, 0x43, 0xed, 0xc0, 0x91, 0x7c, 0xe1, 0xae, 0xfb, 0xfd, 0x32, 0x73, 0x0f, 0x8a, 0xec, - 0x59, 0xa3, 0x0f, 0x9b, 0x3d, 0x6b, 0x45, 0xe6, 0xbc, 0x2a, 0xe5, 0x9b, 0xea, 0xb3, 0x94, 0x56, - 0x7d, 0x32, 0x5d, 0xdd, 0xd1, 0xf3, 0x84, 0x95, 0xf3, 0x4f, 0x02, 0x95, 0x02, 0x6c, 0xc0, 0xec, - 0x60, 0xdf, 0x67, 0xc1, 0x99, 0x76, 0x56, 0xca, 0x3c, 0xf1, 0xd6, 0xf4, 0xf2, 0xc0, 0x39, 0x01, - 0x8d, 0x06, 0x99, 0x94, 0x9f, 0x89, 0x86, 0xb3, 0x9b, 0xa3, 0x03, 0x1d, 0x6e, 0xb8, 0x22, 0xbd, - 0xd5, 0x53, 0x39, 0x69, 0xc6, 0x7a, 0x24, 0x17, 0x5b, 0xcf, 0x48, 0x69, 0xf5, 0xd1, 0xbc, 0x94, - 0x56, 0x03, 0x27, 0xb2, 0x7a, 0x43, 0x25, 0x18, 0x9b, 0xc8, 0x5f, 0x4a, 0x3c, 0x7d, 0x58, 0xdf, - 0xb4, 0x62, 0x6f, 0xa8, 0xb4, 0x62, 0x3d, 0x22, 0xe0, 0xf1, 0xa4, 0x61, 0x7d, 0x93, 0x89, 0x69, - 0x09, 0xc1, 0xa6, 0x8e, 0x27, 0x21, 0x98, 0x71, 0xd5, 0xf0, 0x9c, 0x54, 0xcf, 0xf6, 0xb9, 0x6a, - 0x0c, 0xba, 0xbd, 0x2f, 0x1b, 0x9e, 0xfc, 0x6c, 0xe6, 0xa1, 0x92, 0x9f, 0xdd, 0xd1, 0x93, 0x89, - 0xa1, 0x3e, 0xd9, 0xb2, 0x28, 0xd2, 0x80, 0x29, 0xc4, 0xee, 0xe8, 0x17, 0xe0, 0xa9, 0x7c, 0xba, - 0xea, 0x9e, 0xeb, 0xa6, 0x9b, 0x79, 0x05, 0x76, 0xa5, 0x26, 0x3b, 0x7d, 0x32, 0xa9, 0xc9, 0xce, - 0x1c, 0x7b, 0x6a, 0xb2, 0xc7, 0x4e, 0x20, 0x35, 0xd9, 0xe3, 0x1f, 0x68, 0x6a, 0xb2, 0xd9, 0x47, - 0x90, 0x9a, 0x6c, 0x2d, 0x49, 0x4d, 0x76, 0x36, 0x7f, 0x4a, 0x32, 0xec, 0x87, 0x73, 0x12, 0x92, - 0xdd, 0x61, 0x46, 0x04, 0x3c, 0x02, 0x87, 0x08, 0xd1, 0x97, 0x9d, 0x86, 0x39, 0x2b, 0x4c, 0x07, - 0x9f, 0x12, 0x05, 0xc2, 0x09, 0x29, 0x4a, 0x37, 0x49, 0x50, 0xf6, 0x44, 0x0f, 0x3d, 0x6e, 0x96, - 0x86, 0xac, 0x47, 0x5a, 0xb2, 0xd7, 0x79, 0x5a, 0xb2, 0x73, 0xf9, 0x27, 0x79, 0xfa, 0xba, 0x33, - 0x93, 0x91, 0x7d, 0x7f, 0x01, 0x2e, 0xf4, 0xde, 0x17, 0x89, 0x7a, 0xae, 0x9e, 0x3c, 0x27, 0xa5, - 0xd4, 0x73, 0x5c, 0xb6, 0x4a, 0xb0, 0x06, 0x0e, 0x73, 0x74, 0x0d, 0x66, 0x94, 0xe1, 0x71, 0xcb, - 0x6b, 0xee, 0x6b, 0xe9, 0x9d, 0x95, 0x83, 0x65, 0x23, 0x8d, 0x80, 0xbb, 0xeb, 0xa0, 0x05, 0x98, - 0x32, 0x0a, 0x6b, 0x55, 0x21, 0x43, 0x29, 0x7d, 0x60, 0xc3, 0x04, 0xe3, 0x34, 0xbe, 0xfd, 0x53, - 0x16, 0x3c, 0x9e, 0x93, 0xf5, 0x63, 0xe0, 0x28, 0x3e, 0x9b, 0x30, 0xd5, 0x36, 0xab, 0xf6, 0x09, - 0xf6, 0x65, 0xe4, 0x16, 0x51, 0x7d, 0x4d, 0x01, 0x70, 0x9a, 0xa8, 0x5d, 0x81, 0xf3, 0xbd, 0x6d, - 0x50, 0x2e, 0xff, 0xc6, 0xef, 0x5e, 0xf8, 0xc8, 0x6f, 0xfe, 0xee, 0x85, 0x8f, 0xfc, 0xd6, 0xef, - 0x5e, 0xf8, 0xc8, 0x77, 0x3f, 0xb8, 0x60, 0xfd, 0xc6, 0x83, 0x0b, 0xd6, 0x6f, 0x3e, 0xb8, 0x60, - 0xfd, 0xd6, 0x83, 0x0b, 0xd6, 0xef, 0x3c, 0xb8, 0x60, 0x7d, 0xe5, 0xf7, 0x2e, 0x7c, 0xe4, 0xad, - 0xc2, 0xde, 0x0b, 0xff, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x04, 0xe1, 0xa5, 0xbc, 0x39, 0xec, 0x00, - 0x00, + 0xd3, 0x07, 0xa0, 0x30, 0x6b, 0x13, 0xd0, 0x23, 0x26, 0xc3, 0xb9, 0x04, 0x03, 0xed, 0x88, 0xc8, + 0x48, 0x58, 0xea, 0xfc, 0x66, 0x0a, 0x6d, 0x06, 0xa1, 0xac, 0x69, 0x43, 0xe9, 0x92, 0x35, 0xd6, + 0x94, 0x2b, 0x88, 0x39, 0x8c, 0x76, 0x2e, 0x26, 0xbe, 0xe3, 0xc7, 0x82, 0x81, 0x4d, 0xe2, 0xb7, + 0xb0, 0x52, 0x2c, 0xa0, 0xf6, 0x57, 0x8a, 0x70, 0x2e, 0xd7, 0x10, 0x9f, 0x76, 0x7d, 0x27, 0xf0, + 0xbd, 0x38, 0x50, 0x8f, 0xa2, 0x3c, 0x66, 0x0b, 0x69, 0x6d, 0xad, 0x8a, 0x72, 0xac, 0x30, 0xd0, + 0x65, 0x99, 0x60, 0x3f, 0x9d, 0xcc, 0x68, 0xa1, 0x62, 0xe4, 0xd8, 0xef, 0x37, 0x51, 0xdc, 0x93, + 0x30, 0xd0, 0x0a, 0x82, 0x66, 0xfa, 0xd0, 0xa2, 0xdd, 0x0d, 0x82, 0x26, 0x66, 0x40, 0xf4, 0x09, + 0x31, 0x5e, 0xa9, 0x57, 0x40, 0xec, 0xb8, 0x41, 0xa4, 0x0d, 0xda, 0xd3, 0x30, 0xbc, 0x4d, 0xf6, + 0x42, 0xcf, 0x6f, 0xa4, 0x5f, 0x87, 0x6f, 0xf2, 0x62, 0x2c, 0xe1, 0x66, 0x6a, 0x8b, 0xe1, 0xe3, + 0xce, 0xf0, 0x36, 0xd2, 0xf3, 0x0a, 0xfc, 0x81, 0x22, 0x4c, 0xe2, 0x85, 0xca, 0x37, 0x27, 0xe2, + 0x4e, 0xe7, 0x44, 0x1c, 0x77, 0x86, 0xb7, 0xde, 0xb3, 0xf1, 0x0b, 0x16, 0x4c, 0xb2, 0xf0, 0xcf, + 0x22, 0x52, 0x88, 0x17, 0xf8, 0x27, 0xc0, 0xe2, 0x3d, 0x09, 0x83, 0x21, 0x6d, 0x34, 0x9d, 0xc5, + 0x88, 0xf5, 0x04, 0x73, 0x18, 0x3a, 0x0f, 0x03, 0xac, 0x0b, 0x74, 0xf2, 0xc6, 0x78, 0x02, 0x88, + 0x8a, 0x13, 0x3b, 0x98, 0x95, 0x32, 0xf7, 0x77, 0x4c, 0x5a, 0x4d, 0x8f, 0x77, 0x3a, 0x79, 0x02, + 0xf9, 0x68, 0xb8, 0xbf, 0x67, 0x76, 0xed, 0x83, 0xb9, 0xbf, 0x67, 0x93, 0xec, 0x2e, 0x3e, 0xfd, + 0x61, 0x01, 0x2e, 0x66, 0xd6, 0xeb, 0xdb, 0xfd, 0xbd, 0x7b, 0xed, 0xe3, 0x31, 0xf2, 0xc9, 0xb6, + 0xbd, 0x29, 0x9e, 0xa0, 0xed, 0xcd, 0x40, 0xbf, 0x1c, 0xe6, 0x60, 0x1f, 0x5e, 0xe9, 0x99, 0x43, + 0xf6, 0x11, 0xf1, 0x4a, 0xcf, 0xec, 0x5b, 0x8e, 0xf8, 0xf7, 0x67, 0x85, 0x9c, 0x6f, 0x61, 0x82, + 0xe0, 0x15, 0x7a, 0xce, 0x30, 0x60, 0x24, 0x38, 0xe6, 0x31, 0x7e, 0xc6, 0xf0, 0x32, 0xac, 0xa0, + 0xc8, 0xd3, 0xfc, 0xbb, 0x0b, 0xf9, 0x49, 0x3d, 0x73, 0x9b, 0x9a, 0x33, 0x5f, 0xac, 0xd4, 0x10, + 0x64, 0xf8, 0x7a, 0xaf, 0x6a, 0xc2, 0x7b, 0xb1, 0x7f, 0xe1, 0x7d, 0x2c, 0x5b, 0x70, 0x47, 0xf3, + 0x30, 0xb9, 0xe3, 0xf9, 0xf4, 0xd8, 0xdc, 0x33, 0x59, 0x56, 0x15, 0xee, 0x64, 0xd5, 0x04, 0xe3, + 0x34, 0xfe, 0xec, 0xab, 0x30, 0xfe, 0xf0, 0x6a, 0xcb, 0x6f, 0x14, 0xe1, 0xf1, 0x2e, 0xdb, 0x9e, + 0x9f, 0xf5, 0xc6, 0x1c, 0x68, 0x67, 0x7d, 0xc7, 0x3c, 0x54, 0xe1, 0xf4, 0x66, 0xbb, 0xd9, 0xdc, + 0x63, 0xe6, 0xad, 0xc4, 0x95, 0x18, 0x82, 0xa7, 0x3c, 0x2f, 0x53, 0x6e, 0x2c, 0x67, 0xe0, 0xe0, + 0xcc, 0x9a, 0xe8, 0x0d, 0x40, 0x81, 0xc8, 0x28, 0x7c, 0x9d, 0xf8, 0xe2, 0x1d, 0x80, 0x0d, 0x7c, + 0x31, 0xd9, 0x8c, 0xb7, 0x3b, 0x30, 0x70, 0x46, 0x2d, 0x2a, 0x1c, 0xd0, 0x5b, 0x69, 0x4f, 0x75, + 0x2b, 0x25, 0x1c, 0x60, 0x1d, 0x88, 0x4d, 0x5c, 0x74, 0x1d, 0xa6, 0x9d, 0x5d, 0xc7, 0xe3, 0x61, + 0x00, 0x25, 0x01, 0x2e, 0x1d, 0x28, 0x65, 0xd9, 0x7c, 0x1a, 0x01, 0x77, 0xd6, 0x49, 0x79, 0x80, + 0x0f, 0xe5, 0x7b, 0x80, 0x77, 0x3f, 0x17, 0x7b, 0xe9, 0x7e, 0xed, 0xff, 0x6a, 0xd1, 0xeb, 0x4b, + 0x4b, 0xf4, 0xaf, 0x1e, 0x5b, 0x5f, 0x65, 0x06, 0x2c, 0x5c, 0x19, 0xa8, 0x39, 0x63, 0x9f, 0xd1, + 0x0c, 0x58, 0x12, 0x20, 0x36, 0x71, 0xf9, 0x82, 0x88, 0x12, 0x1f, 0x20, 0x83, 0xc5, 0x17, 0xc1, + 0x1c, 0x14, 0x06, 0xfa, 0x3c, 0x0c, 0xbb, 0xde, 0xae, 0x17, 0x05, 0xa1, 0xd8, 0x2c, 0x47, 0xf4, + 0xa4, 0x48, 0xce, 0xc1, 0x0a, 0x27, 0x83, 0x25, 0x3d, 0xfb, 0x07, 0x0a, 0x30, 0x2e, 0x5b, 0x7c, + 0xb3, 0x1d, 0xc4, 0xce, 0x09, 0x5c, 0xcb, 0xd7, 0x8d, 0x6b, 0xf9, 0x13, 0xdd, 0x22, 0x5a, 0xb0, + 0x2e, 0xe5, 0x5e, 0xc7, 0xb7, 0x53, 0xd7, 0xf1, 0x53, 0xbd, 0x49, 0x75, 0xbf, 0x86, 0xff, 0x85, + 0x05, 0xd3, 0x06, 0xfe, 0x09, 0xdc, 0x06, 0xcb, 0xe6, 0x6d, 0xf0, 0x44, 0xcf, 0x6f, 0xc8, 0xb9, + 0x05, 0xbe, 0xb7, 0x98, 0xea, 0x3b, 0x3b, 0xfd, 0xdf, 0x83, 0x81, 0x2d, 0x27, 0x74, 0xbb, 0x45, + 0xce, 0xed, 0xa8, 0x34, 0x77, 0xc3, 0x09, 0x5d, 0x7e, 0x86, 0x3f, 0xab, 0xd2, 0x72, 0x3a, 0xa1, + 0xdb, 0xd3, 0xe5, 0x8d, 0x35, 0x85, 0x5e, 0x81, 0xa1, 0xa8, 0x1e, 0xb4, 0x94, 0x41, 0xea, 0x25, + 0x9e, 0xb2, 0x93, 0x96, 0x1c, 0xee, 0x97, 0x91, 0xd9, 0x1c, 0x2d, 0xc6, 0x02, 0x1f, 0xbd, 0x05, + 0xe3, 0xec, 0x97, 0xb2, 0x94, 0x28, 0xe6, 0xe7, 0x6b, 0xa8, 0xe9, 0x88, 0xdc, 0xe0, 0xc6, 0x28, + 0xc2, 0x26, 0xa9, 0xd9, 0x06, 0x94, 0xd4, 0x67, 0x3d, 0x52, 0x57, 0xa5, 0xff, 0x54, 0x84, 0x53, + 0x19, 0x6b, 0x0e, 0x45, 0xc6, 0x4c, 0x3c, 0xdf, 0xe7, 0x52, 0xfd, 0x80, 0x73, 0x11, 0x31, 0x69, + 0xc8, 0x15, 0x6b, 0xab, 0xef, 0x46, 0xef, 0x44, 0x24, 0xdd, 0x28, 0x2d, 0xea, 0xdd, 0x28, 0x6d, + 0xec, 0xc4, 0x86, 0x9a, 0x36, 0xa4, 0x7a, 0xfa, 0x48, 0xe7, 0xf4, 0x8f, 0x8b, 0x70, 0x3a, 0x2b, + 0xc8, 0x0e, 0xfa, 0xce, 0x54, 0xee, 0x9e, 0x17, 0xfb, 0x0d, 0xcf, 0xc3, 0x13, 0xfa, 0x88, 0xd4, + 0xdb, 0x73, 0x66, 0x36, 0x9f, 0x9e, 0xc3, 0x2c, 0xda, 0x64, 0xfe, 0xad, 0x21, 0xcf, 0xb9, 0x24, + 0x8f, 0x8f, 0x4f, 0xf7, 0xdd, 0x01, 0x91, 0xac, 0x29, 0x4a, 0xf9, 0xb7, 0xca, 0xe2, 0xde, 0xfe, + 0xad, 0xb2, 0xe5, 0x59, 0x0f, 0x46, 0xb5, 0xaf, 0x79, 0xa4, 0x33, 0xbe, 0x4d, 0x6f, 0x2b, 0xad, + 0xdf, 0x8f, 0x74, 0xd6, 0x7f, 0xd8, 0x82, 0x94, 0xf5, 0xa7, 0x52, 0x8b, 0x59, 0xb9, 0x6a, 0xb1, + 0x4b, 0x30, 0x10, 0x06, 0x4d, 0x92, 0x4e, 0x95, 0x83, 0x83, 0x26, 0xc1, 0x0c, 0x42, 0x31, 0xe2, + 0x44, 0xd9, 0x31, 0xa6, 0x0b, 0x72, 0x42, 0x44, 0x7b, 0x12, 0x06, 0x9b, 0x64, 0x97, 0x34, 0xd3, + 0x71, 0xe8, 0x6f, 0xd1, 0x42, 0xcc, 0x61, 0xf6, 0x2f, 0x0c, 0xc0, 0x85, 0xae, 0x1e, 0xe2, 0x54, + 0x1c, 0x6a, 0x38, 0x31, 0xb9, 0xef, 0xec, 0xa5, 0x03, 0x46, 0x5f, 0xe7, 0xc5, 0x58, 0xc2, 0x99, + 0x41, 0x3c, 0x0f, 0x10, 0x99, 0x52, 0x22, 0x8a, 0xb8, 0x90, 0x02, 0x6a, 0x2a, 0xa5, 0x8a, 0xc7, + 0xa1, 0x94, 0xba, 0x06, 0x10, 0x45, 0x4d, 0x6e, 0x5f, 0xe0, 0x0a, 0x4b, 0xfb, 0x24, 0x90, 0x68, + 0xed, 0x96, 0x80, 0x60, 0x0d, 0x0b, 0x55, 0x60, 0xaa, 0x15, 0x06, 0x31, 0xd7, 0xc9, 0x56, 0xb8, + 0x61, 0xd2, 0xa0, 0xe9, 0x9c, 0x5b, 0x4d, 0xc1, 0x71, 0x47, 0x0d, 0xf4, 0x12, 0x8c, 0x0a, 0x87, + 0xdd, 0x6a, 0x10, 0x34, 0x85, 0x1a, 0x48, 0x99, 0xb9, 0xd4, 0x12, 0x10, 0xd6, 0xf1, 0xb4, 0x6a, + 0x4c, 0xd1, 0x3b, 0x9c, 0x59, 0x8d, 0x2b, 0x7b, 0x35, 0xbc, 0x54, 0xc0, 0xad, 0x91, 0xbe, 0x02, + 0x6e, 0x25, 0x8a, 0xb1, 0x52, 0xdf, 0x6f, 0x5b, 0xd0, 0x53, 0x95, 0xf4, 0xb3, 0x03, 0x70, 0x4a, + 0x2c, 0x9c, 0x47, 0xbd, 0x5c, 0xee, 0x74, 0x2e, 0x97, 0xe3, 0x50, 0x9d, 0x7d, 0x73, 0xcd, 0x9c, + 0xf4, 0x9a, 0xf9, 0x41, 0x0b, 0x4c, 0xf6, 0x0a, 0xfd, 0xa5, 0xdc, 0x88, 0xfb, 0x2f, 0xe5, 0xb2, + 0x6b, 0xae, 0xbc, 0x40, 0x3e, 0x60, 0xec, 0x7d, 0xfb, 0xbf, 0x58, 0xf0, 0x44, 0x4f, 0x8a, 0x68, + 0x09, 0x4a, 0x8c, 0x07, 0xd4, 0xa4, 0xb3, 0xa7, 0x94, 0xe1, 0xa2, 0x04, 0xe4, 0xb0, 0xa4, 0x49, + 0x4d, 0xb4, 0xd4, 0x91, 0xda, 0xe0, 0xe9, 0x8c, 0xd4, 0x06, 0x67, 0x8c, 0xe1, 0x79, 0xc8, 0xdc, + 0x06, 0xbf, 0x5c, 0x84, 0x21, 0xbe, 0xe2, 0x4f, 0x40, 0x0c, 0x5b, 0x16, 0x7a, 0xdb, 0x2e, 0x21, + 0xb7, 0x78, 0x5f, 0xe6, 0x2a, 0x4e, 0xec, 0x70, 0x36, 0x41, 0xdd, 0x56, 0x89, 0x86, 0x17, 0xcd, + 0x19, 0xf7, 0xd9, 0x6c, 0x4a, 0x31, 0x09, 0x9c, 0x86, 0x76, 0xbb, 0x7d, 0x11, 0x20, 0x8a, 0x43, + 0xcf, 0x6f, 0x50, 0x1a, 0x22, 0x78, 0xdb, 0x27, 0xbb, 0xb4, 0x5e, 0x53, 0xc8, 0xbc, 0x0f, 0xc9, + 0x4e, 0x57, 0x00, 0xac, 0x51, 0x9c, 0x7d, 0x19, 0x4a, 0x0a, 0xb9, 0x97, 0x16, 0x67, 0x4c, 0x67, + 0x2e, 0x3e, 0x0b, 0x93, 0xa9, 0xb6, 0x8e, 0xa4, 0x04, 0xfa, 0x45, 0x0b, 0x26, 0x79, 0x97, 0x97, + 0xfc, 0x5d, 0x71, 0xa6, 0xbe, 0x0f, 0xa7, 0x9b, 0x19, 0x67, 0x9b, 0x98, 0xd1, 0xfe, 0xcf, 0x42, + 0xa5, 0xf4, 0xc9, 0x82, 0xe2, 0xcc, 0x36, 0xd0, 0x15, 0xba, 0x6e, 0xe9, 0xd9, 0xe5, 0x34, 0x85, + 0x73, 0xd5, 0x18, 0x5f, 0xb3, 0xbc, 0x0c, 0x2b, 0xa8, 0xfd, 0x5b, 0x16, 0x4c, 0xf3, 0x9e, 0xdf, + 0x24, 0x7b, 0x6a, 0x87, 0x7f, 0x98, 0x7d, 0x17, 0xd9, 0x46, 0x0a, 0x39, 0xd9, 0x46, 0xf4, 0x4f, + 0x2b, 0x76, 0xfd, 0xb4, 0x9f, 0xb6, 0x40, 0xac, 0xc0, 0x13, 0x10, 0xe5, 0xbf, 0xd5, 0x14, 0xe5, + 0x67, 0xf3, 0x17, 0x75, 0x8e, 0x0c, 0xff, 0xa7, 0x16, 0x4c, 0x71, 0x84, 0xe4, 0xcd, 0xf9, 0x43, + 0x9d, 0x87, 0x7e, 0xd2, 0x06, 0xaa, 0x5c, 0xe2, 0xd9, 0x1f, 0x65, 0x4c, 0xd6, 0x40, 0xd7, 0xc9, + 0x72, 0xe5, 0x06, 0x3a, 0x42, 0xca, 0xcc, 0x23, 0x47, 0xed, 0xb6, 0xff, 0xc0, 0x02, 0xc4, 0x9b, + 0x31, 0xd8, 0x1f, 0xca, 0x54, 0xb0, 0x52, 0xed, 0xba, 0x48, 0x8e, 0x1a, 0x05, 0xc1, 0x1a, 0xd6, + 0xb1, 0x0c, 0x4f, 0xca, 0x70, 0xa0, 0xd8, 0xdb, 0x70, 0xe0, 0x08, 0x23, 0xfa, 0xfb, 0x83, 0x90, + 0x76, 0x3f, 0x40, 0x77, 0x61, 0xac, 0xee, 0xb4, 0x9c, 0x0d, 0xaf, 0xe9, 0xc5, 0x1e, 0x89, 0xba, + 0x59, 0x1c, 0x2d, 0x6a, 0x78, 0xe2, 0xa9, 0x57, 0x2b, 0xc1, 0x06, 0x1d, 0x34, 0x07, 0xd0, 0x0a, + 0xbd, 0x5d, 0xaf, 0x49, 0x1a, 0x4c, 0xe3, 0xc0, 0xdc, 0x39, 0xb9, 0x19, 0x8d, 0x2c, 0xc5, 0x1a, + 0x46, 0x86, 0x67, 0x5e, 0xf1, 0xd1, 0x79, 0xe6, 0x0d, 0x1c, 0xd1, 0x33, 0x6f, 0xb0, 0x2f, 0xcf, + 0x3c, 0x0c, 0x67, 0x25, 0x8b, 0x44, 0xff, 0x2f, 0x7b, 0x4d, 0x22, 0xf8, 0x62, 0xee, 0xe4, 0x39, + 0x7b, 0xb0, 0x5f, 0x3e, 0x8b, 0x33, 0x31, 0x70, 0x4e, 0x4d, 0xf4, 0x39, 0x98, 0x71, 0x9a, 0xcd, + 0xe0, 0xbe, 0x1a, 0xb5, 0xa5, 0xa8, 0xee, 0x34, 0xb9, 0xc6, 0x7e, 0x98, 0x51, 0x3d, 0x7f, 0xb0, + 0x5f, 0x9e, 0x99, 0xcf, 0xc1, 0xc1, 0xb9, 0xb5, 0x53, 0x8e, 0x7d, 0x23, 0x3d, 0x1d, 0xfb, 0x5e, + 0x83, 0x52, 0x2b, 0x0c, 0xea, 0xab, 0x9a, 0xf7, 0xcf, 0x45, 0x96, 0x90, 0x5f, 0x16, 0x1e, 0xee, + 0x97, 0xc7, 0xd5, 0x1f, 0x76, 0xc3, 0x27, 0x15, 0x32, 0xfc, 0xf9, 0xe0, 0x51, 0xfa, 0xf3, 0x6d, + 0xc3, 0xa9, 0x1a, 0x09, 0x3d, 0x96, 0x59, 0xd4, 0x4d, 0xce, 0x8f, 0x75, 0x28, 0x85, 0xa9, 0x13, + 0xb3, 0xaf, 0x30, 0x55, 0x5a, 0xf4, 0x64, 0x79, 0x42, 0x26, 0x84, 0xec, 0x3f, 0xb1, 0x60, 0x58, + 0x18, 0xbe, 0x9f, 0x00, 0xa3, 0x36, 0x6f, 0xe8, 0xcb, 0xcb, 0xd9, 0xb7, 0x0a, 0xeb, 0x4c, 0xae, + 0xa6, 0x7c, 0x25, 0xa5, 0x29, 0x7f, 0xa2, 0x1b, 0x91, 0xee, 0x3a, 0xf2, 0xbf, 0x5b, 0x84, 0x09, + 0xd3, 0x57, 0xe5, 0x04, 0x86, 0x60, 0x0d, 0x86, 0x23, 0xe1, 0x18, 0x55, 0xc8, 0x37, 0xe8, 0x4e, + 0x4f, 0x62, 0x62, 0xad, 0x25, 0x5c, 0xa1, 0x24, 0x91, 0x4c, 0x8f, 0xab, 0xe2, 0x23, 0xf4, 0xb8, + 0xea, 0xe5, 0x2e, 0x34, 0x70, 0x1c, 0xee, 0x42, 0xf6, 0xd7, 0xd8, 0xcd, 0xa6, 0x97, 0x9f, 0x00, + 0xd3, 0x73, 0xdd, 0xbc, 0x03, 0xed, 0x2e, 0x2b, 0x4b, 0x74, 0x2a, 0x87, 0xf9, 0xf9, 0x79, 0x0b, + 0x2e, 0x64, 0x7c, 0x95, 0xc6, 0x09, 0x3d, 0x0b, 0x23, 0x4e, 0xdb, 0xf5, 0xd4, 0x5e, 0xd6, 0x5e, + 0xcd, 0xe6, 0x45, 0x39, 0x56, 0x18, 0x68, 0x11, 0xa6, 0xc9, 0x83, 0x96, 0xc7, 0x9f, 0x2d, 0x75, + 0x93, 0xca, 0x22, 0x0f, 0xdd, 0xbb, 0x94, 0x06, 0xe2, 0x4e, 0x7c, 0xe5, 0xdc, 0x5e, 0xcc, 0x75, + 0x6e, 0xff, 0xc7, 0x16, 0x8c, 0x2a, 0x27, 0x98, 0x47, 0x3e, 0xda, 0xdf, 0x66, 0x8e, 0xf6, 0xe3, + 0x5d, 0x46, 0x3b, 0x67, 0x98, 0xff, 0x7e, 0x41, 0xf5, 0xb7, 0x1a, 0x84, 0x71, 0x1f, 0x1c, 0xd6, + 0x2b, 0x30, 0xd2, 0x0a, 0x83, 0x38, 0xa8, 0x07, 0x4d, 0xc1, 0x60, 0x9d, 0x4f, 0x62, 0x2f, 0xf0, + 0xf2, 0x43, 0xed, 0x37, 0x56, 0xd8, 0x6c, 0xf4, 0x82, 0x30, 0x16, 0x4c, 0x4d, 0x32, 0x7a, 0x41, + 0x18, 0x63, 0x06, 0x41, 0x2e, 0x40, 0xec, 0x84, 0x0d, 0x12, 0xd3, 0x32, 0x11, 0xc6, 0x25, 0xff, + 0xf0, 0x68, 0xc7, 0x5e, 0x73, 0xce, 0xf3, 0xe3, 0x28, 0x0e, 0xe7, 0x56, 0xfc, 0xf8, 0x76, 0xc8, + 0xe5, 0x35, 0x2d, 0x98, 0x82, 0xa2, 0x85, 0x35, 0xba, 0xd2, 0x05, 0x95, 0xb5, 0x31, 0x68, 0xbe, + 0xbf, 0xaf, 0x89, 0x72, 0xac, 0x30, 0xec, 0x97, 0xd9, 0x55, 0xc2, 0x06, 0xe8, 0x68, 0x71, 0x0e, + 0xbe, 0x3e, 0xa2, 0x86, 0x96, 0x3d, 0xbe, 0x55, 0xf4, 0x68, 0x0a, 0xdd, 0x4f, 0x6e, 0xda, 0xb0, + 0xee, 0xde, 0x93, 0x84, 0x5c, 0x40, 0xdf, 0xde, 0x61, 0x96, 0xf1, 0x5c, 0x8f, 0x2b, 0xe0, 0x08, + 0x86, 0x18, 0x2c, 0x9c, 0x38, 0x0b, 0xb6, 0xbc, 0x52, 0x15, 0x8b, 0x5c, 0x0b, 0x27, 0x2e, 0x00, + 0x38, 0xc1, 0x41, 0x57, 0x85, 0xb4, 0x3f, 0x60, 0x24, 0x15, 0x94, 0xd2, 0xbe, 0xfc, 0x7c, 0x4d, + 0xdc, 0x7f, 0x1e, 0x46, 0x55, 0x72, 0xc1, 0x2a, 0xcf, 0xd1, 0x26, 0x82, 0xda, 0x2c, 0x25, 0xc5, + 0x58, 0xc7, 0x41, 0xeb, 0x30, 0x19, 0x71, 0x55, 0x8f, 0x8a, 0x5d, 0xc8, 0x55, 0x66, 0x9f, 0x94, + 0xe6, 0x1c, 0x35, 0x13, 0x7c, 0xc8, 0x8a, 0xf8, 0xd1, 0x21, 0xfd, 0x48, 0xd3, 0x24, 0xd0, 0xeb, + 0x30, 0xd1, 0xd4, 0xd3, 0xf8, 0x57, 0x85, 0x46, 0x4d, 0x59, 0x45, 0x1b, 0x49, 0xfe, 0xab, 0x38, + 0x85, 0x4d, 0x19, 0x33, 0xbd, 0x44, 0xc4, 0xdb, 0x74, 0xfc, 0x06, 0x89, 0x44, 0x6a, 0x34, 0xc6, + 0x98, 0xdd, 0xca, 0xc1, 0xc1, 0xb9, 0xb5, 0xd1, 0x2b, 0x30, 0x26, 0x3f, 0x5f, 0xf3, 0x92, 0x4e, + 0x6c, 0xef, 0x35, 0x18, 0x36, 0x30, 0xd1, 0x7d, 0x38, 0x23, 0xff, 0xaf, 0x87, 0xce, 0xe6, 0xa6, + 0x57, 0x17, 0x4e, 0xea, 0xdc, 0x01, 0x69, 0x5e, 0x7a, 0x34, 0x2d, 0x65, 0x21, 0x1d, 0xee, 0x97, + 0x2f, 0x89, 0x51, 0xcb, 0x84, 0xb3, 0x49, 0xcc, 0xa6, 0x8f, 0x56, 0xe1, 0xd4, 0x16, 0x71, 0x9a, + 0xf1, 0xd6, 0xe2, 0x16, 0xa9, 0x6f, 0xcb, 0x4d, 0xc4, 0x7c, 0xaf, 0x35, 0x8b, 0xf5, 0x1b, 0x9d, + 0x28, 0x38, 0xab, 0x1e, 0x7a, 0x1b, 0x66, 0x5a, 0xed, 0x8d, 0xa6, 0x17, 0x6d, 0xad, 0x05, 0x31, + 0xb3, 0x20, 0x51, 0xb9, 0xf9, 0x84, 0x93, 0xb6, 0xf2, 0x3b, 0xaf, 0xe6, 0xe0, 0xe1, 0x5c, 0x0a, + 0xe8, 0x7d, 0x38, 0x93, 0x5a, 0x0c, 0xc2, 0x65, 0x74, 0x22, 0x3f, 0x7a, 0x71, 0x2d, 0xab, 0x82, + 0x70, 0x01, 0xcd, 0x02, 0xe1, 0xec, 0x26, 0x3e, 0x98, 0x5d, 0xd1, 0x7b, 0xb4, 0xb2, 0xc6, 0x94, + 0xa1, 0x77, 0x60, 0x4c, 0x5f, 0x45, 0xe2, 0x82, 0xb9, 0x9c, 0xcd, 0xb3, 0x68, 0xab, 0x8d, 0xb3, + 0x74, 0x6a, 0x45, 0xe9, 0x30, 0x6c, 0x50, 0xb4, 0x09, 0x64, 0x7f, 0x1f, 0xba, 0x05, 0x23, 0xf5, + 0xa6, 0x47, 0xfc, 0x78, 0xa5, 0xda, 0x2d, 0x84, 0xca, 0xa2, 0xc0, 0x11, 0x03, 0x26, 0xc2, 0xbd, + 0xf2, 0x32, 0xac, 0x28, 0xd8, 0xbf, 0x5a, 0x80, 0x72, 0x8f, 0xd8, 0xc1, 0x29, 0xf5, 0xb7, 0xd5, + 0x97, 0xfa, 0x7b, 0x5e, 0x66, 0x1a, 0x5c, 0x4b, 0xe9, 0x04, 0x52, 0x59, 0x04, 0x13, 0xcd, 0x40, + 0x1a, 0xbf, 0x6f, 0x73, 0x64, 0x5d, 0x83, 0x3e, 0xd0, 0xd3, 0xa0, 0xde, 0x78, 0x39, 0x1b, 0xec, + 0x5f, 0x10, 0xc9, 0x7d, 0x05, 0xb1, 0xbf, 0x56, 0x80, 0x33, 0x6a, 0x08, 0xff, 0xe2, 0x0e, 0xdc, + 0x9d, 0xce, 0x81, 0x3b, 0x86, 0x37, 0x24, 0xfb, 0x36, 0x0c, 0xf1, 0x10, 0x34, 0x7d, 0x30, 0x40, + 0x4f, 0x9a, 0xf1, 0xca, 0xd4, 0x35, 0x6d, 0xc4, 0x2c, 0xfb, 0x6b, 0x16, 0x4c, 0xae, 0x2f, 0x56, + 0x6b, 0x41, 0x7d, 0x9b, 0xc4, 0xf3, 0x9c, 0x61, 0xc5, 0x82, 0xff, 0xb1, 0x1e, 0x92, 0xaf, 0xc9, + 0xe2, 0x98, 0x2e, 0xc1, 0xc0, 0x56, 0x10, 0xc5, 0xe9, 0x07, 0xe6, 0x1b, 0x41, 0x14, 0x63, 0x06, + 0xb1, 0x7f, 0xdb, 0x82, 0x41, 0x96, 0x1f, 0xb7, 0x57, 0xd2, 0xe6, 0x7e, 0xbe, 0x0b, 0xbd, 0x04, + 0x43, 0x64, 0x73, 0x93, 0xd4, 0x63, 0x31, 0xab, 0xd2, 0x4b, 0x76, 0x68, 0x89, 0x95, 0xd2, 0x4b, + 0x9f, 0x35, 0xc6, 0xff, 0x62, 0x81, 0x8c, 0xee, 0x41, 0x29, 0xf6, 0x76, 0xc8, 0xbc, 0xeb, 0x8a, + 0x27, 0xba, 0x87, 0x70, 0x4a, 0x5e, 0x97, 0x04, 0x70, 0x42, 0xcb, 0xfe, 0x4a, 0x01, 0x20, 0x89, + 0x6c, 0xd0, 0xeb, 0x13, 0x17, 0x3a, 0x1e, 0x6f, 0x2e, 0x67, 0x3c, 0xde, 0xa0, 0x84, 0x60, 0xc6, + 0xcb, 0x8d, 0x1a, 0xa6, 0x62, 0x5f, 0xc3, 0x34, 0x70, 0x94, 0x61, 0x5a, 0x84, 0xe9, 0x24, 0x32, + 0x83, 0x19, 0xa6, 0x86, 0x09, 0x29, 0xeb, 0x69, 0x20, 0xee, 0xc4, 0xb7, 0x09, 0x5c, 0x92, 0xf1, + 0x49, 0xe5, 0x5d, 0xc3, 0x2c, 0x40, 0x8f, 0x90, 0xbf, 0x3b, 0x79, 0x9d, 0x2a, 0xe4, 0xbe, 0x4e, + 0xfd, 0x98, 0x05, 0xa7, 0xd3, 0xed, 0x30, 0x97, 0xbc, 0x2f, 0x5b, 0x70, 0x86, 0xbd, 0xd1, 0xb1, + 0x56, 0x3b, 0x5f, 0x04, 0x5f, 0xcc, 0x8e, 0x58, 0xd1, 0xbd, 0xc7, 0x89, 0x3b, 0xf6, 0x6a, 0x16, + 0x69, 0x9c, 0xdd, 0xa2, 0xfd, 0x65, 0x0b, 0xce, 0xe5, 0xa6, 0x65, 0x42, 0x57, 0x60, 0xc4, 0x69, + 0x79, 0x5c, 0x01, 0x26, 0xf6, 0x3b, 0x93, 0x1e, 0xab, 0x2b, 0x5c, 0xfd, 0xa5, 0xa0, 0x2a, 0x5d, + 0x64, 0x21, 0x37, 0x5d, 0x64, 0xcf, 0xec, 0x8f, 0xf6, 0xf7, 0x59, 0x20, 0xbc, 0xb0, 0xfa, 0x38, + 0x64, 0xde, 0x92, 0xd9, 0x76, 0x8d, 0xd0, 0xf0, 0x97, 0xf2, 0xdd, 0xd2, 0x44, 0x40, 0x78, 0x75, + 0xa9, 0x1b, 0x61, 0xe0, 0x0d, 0x5a, 0xb6, 0x0b, 0x02, 0x5a, 0x21, 0x4c, 0x67, 0xd5, 0xbb, 0x37, + 0xd7, 0x00, 0x5c, 0x86, 0xab, 0xe5, 0xdc, 0x54, 0x57, 0x48, 0x45, 0x41, 0xb0, 0x86, 0x65, 0xff, + 0x87, 0x02, 0x8c, 0xca, 0x50, 0xe4, 0x6d, 0xbf, 0x1f, 0xc9, 0xf2, 0x48, 0xb9, 0x89, 0x58, 0x92, + 0x5a, 0x4a, 0xb8, 0x9a, 0x08, 0xe4, 0x49, 0x92, 0x5a, 0x09, 0xc0, 0x09, 0x0e, 0x7a, 0x1a, 0x86, + 0xa3, 0xf6, 0x06, 0x43, 0x4f, 0xf9, 0x0c, 0xd5, 0x78, 0x31, 0x96, 0x70, 0xf4, 0x39, 0x98, 0xe2, + 0xf5, 0xc2, 0xa0, 0xe5, 0x34, 0xb8, 0xb6, 0x75, 0x50, 0x39, 0xfb, 0x4e, 0xad, 0xa6, 0x60, 0x87, + 0xfb, 0xe5, 0xd3, 0xe9, 0x32, 0xa6, 0xa7, 0xef, 0xa0, 0xc2, 0xde, 0xfe, 0x79, 0x23, 0x74, 0x99, + 0x76, 0x98, 0x0c, 0x24, 0x20, 0xac, 0xe3, 0xd9, 0xef, 0x00, 0xea, 0x0c, 0xca, 0x8e, 0xde, 0xe0, + 0x06, 0x5f, 0x5e, 0x48, 0xdc, 0x6e, 0x7a, 0x7b, 0xdd, 0xa5, 0x55, 0x9a, 0xfb, 0xf3, 0x5a, 0x58, + 0xd5, 0xb7, 0xff, 0x46, 0x11, 0xa6, 0xd2, 0x0e, 0x8e, 0xe8, 0x06, 0x0c, 0xf1, 0x3b, 0x52, 0x90, + 0xef, 0xf2, 0x2c, 0xac, 0xb9, 0x45, 0xb2, 0xd3, 0x42, 0x5c, 0xb3, 0xa2, 0x3e, 0x7a, 0x1b, 0x46, + 0xdd, 0xe0, 0xbe, 0x7f, 0xdf, 0x09, 0xdd, 0xf9, 0xea, 0x8a, 0x58, 0xce, 0x99, 0xac, 0x76, 0x25, + 0x41, 0xd3, 0x5d, 0x2d, 0xd9, 0x13, 0x48, 0x02, 0xc2, 0x3a, 0x39, 0xb4, 0xce, 0x02, 0x4d, 0x6e, + 0x7a, 0x8d, 0x55, 0xa7, 0xd5, 0xcd, 0xfa, 0x77, 0x51, 0x22, 0x69, 0x94, 0xc7, 0x45, 0x34, 0x4a, + 0x0e, 0xc0, 0x09, 0x21, 0xf4, 0x9d, 0x70, 0x2a, 0xca, 0xd1, 0xce, 0xe5, 0xe5, 0xe8, 0xe8, 0xa6, + 0xb0, 0x5a, 0x78, 0x8c, 0x0a, 0x41, 0x59, 0x7a, 0xbc, 0xac, 0x66, 0xec, 0x5f, 0x3b, 0x05, 0xc6, + 0x26, 0x36, 0x52, 0x36, 0x59, 0xc7, 0x94, 0xb2, 0x09, 0xc3, 0x08, 0xd9, 0x69, 0xc5, 0x7b, 0x15, + 0x2f, 0xec, 0x96, 0x52, 0x70, 0x49, 0xe0, 0x74, 0xd2, 0x94, 0x10, 0xac, 0xe8, 0x64, 0xe7, 0xd5, + 0x2a, 0x7e, 0x88, 0x79, 0xb5, 0x06, 0x4e, 0x30, 0xaf, 0xd6, 0x1a, 0x0c, 0x37, 0xbc, 0x18, 0x93, + 0x56, 0x20, 0xb8, 0xd3, 0xcc, 0x75, 0x78, 0x9d, 0xa3, 0x74, 0x66, 0x70, 0x11, 0x00, 0x2c, 0x89, + 0xa0, 0x37, 0xd4, 0x0e, 0x1c, 0xca, 0x17, 0xee, 0x3a, 0xdf, 0x2f, 0x33, 0xf7, 0xa0, 0xc8, 0x9e, + 0x35, 0xfc, 0xb0, 0xd9, 0xb3, 0x96, 0x65, 0xce, 0xab, 0x91, 0x7c, 0x53, 0x7d, 0x96, 0xd2, 0xaa, + 0x47, 0xa6, 0xab, 0xbb, 0x7a, 0x9e, 0xb0, 0x52, 0xfe, 0x49, 0xa0, 0x52, 0x80, 0xf5, 0x99, 0x1d, + 0xec, 0xfb, 0x2c, 0x38, 0xd3, 0xca, 0x4a, 0x99, 0x27, 0xde, 0x9a, 0x5e, 0xea, 0x3b, 0x27, 0xa0, + 0xd1, 0x20, 0x93, 0xf2, 0x33, 0xd1, 0x70, 0x76, 0x73, 0x74, 0xa0, 0xc3, 0x0d, 0x57, 0xa4, 0xb7, + 0x7a, 0x32, 0x27, 0xcd, 0x58, 0x97, 0xe4, 0x62, 0xeb, 0x19, 0x29, 0xad, 0x3e, 0x9e, 0x97, 0xd2, + 0xaa, 0xef, 0x44, 0x56, 0x6f, 0xa8, 0x04, 0x63, 0xe3, 0xf9, 0x4b, 0x89, 0xa7, 0x0f, 0xeb, 0x99, + 0x56, 0xec, 0x0d, 0x95, 0x56, 0xac, 0x4b, 0x04, 0x3c, 0x9e, 0x34, 0xac, 0x67, 0x32, 0x31, 0x2d, + 0x21, 0xd8, 0xe4, 0xf1, 0x24, 0x04, 0x33, 0xae, 0x1a, 0x9e, 0x93, 0xea, 0x99, 0x1e, 0x57, 0x8d, + 0x41, 0xb7, 0xfb, 0x65, 0xc3, 0x93, 0x9f, 0x4d, 0x3f, 0x54, 0xf2, 0xb3, 0xbb, 0x7a, 0x32, 0x31, + 0xd4, 0x23, 0x5b, 0x16, 0x45, 0xea, 0x33, 0x85, 0xd8, 0x5d, 0xfd, 0x02, 0x3c, 0x95, 0x4f, 0x57, + 0xdd, 0x73, 0x9d, 0x74, 0x33, 0xaf, 0xc0, 0x8e, 0xd4, 0x64, 0xa7, 0x4f, 0x26, 0x35, 0xd9, 0x99, + 0x63, 0x4f, 0x4d, 0x76, 0xf6, 0x04, 0x52, 0x93, 0x3d, 0xf6, 0xa1, 0xa6, 0x26, 0x9b, 0x79, 0x04, + 0xa9, 0xc9, 0xd6, 0x92, 0xd4, 0x64, 0xe7, 0xf2, 0xa7, 0x24, 0xc3, 0x7e, 0x38, 0x27, 0x21, 0xd9, + 0x5d, 0x66, 0x44, 0xc0, 0x23, 0x70, 0x88, 0x10, 0x7d, 0xd9, 0x69, 0x98, 0xb3, 0xc2, 0x74, 0xf0, + 0x29, 0x51, 0x20, 0x9c, 0x90, 0xa2, 0x74, 0x93, 0x04, 0x65, 0x8f, 0x77, 0xd1, 0xe3, 0x66, 0x69, + 0xc8, 0xba, 0xa4, 0x25, 0x7b, 0x9d, 0xa7, 0x25, 0x3b, 0x9f, 0x7f, 0x92, 0xa7, 0xaf, 0x3b, 0x33, + 0x19, 0xd9, 0xf7, 0x17, 0xe0, 0x62, 0xf7, 0x7d, 0x91, 0xa8, 0xe7, 0xaa, 0xc9, 0x73, 0x52, 0x4a, + 0x3d, 0xc7, 0x65, 0xab, 0x04, 0xab, 0xef, 0x30, 0x47, 0xd7, 0x61, 0x5a, 0x19, 0x1e, 0x37, 0xbd, + 0xfa, 0x9e, 0x96, 0xde, 0x59, 0x39, 0x58, 0xd6, 0xd2, 0x08, 0xb8, 0xb3, 0x0e, 0x9a, 0x87, 0x49, + 0xa3, 0x70, 0xa5, 0x22, 0x64, 0x28, 0xa5, 0x0f, 0xac, 0x99, 0x60, 0x9c, 0xc6, 0xb7, 0x7f, 0xca, + 0x82, 0xc7, 0x72, 0xb2, 0x7e, 0xf4, 0x1d, 0xc5, 0x67, 0x13, 0x26, 0x5b, 0x66, 0xd5, 0x1e, 0xc1, + 0xbe, 0x8c, 0xdc, 0x22, 0xaa, 0xaf, 0x29, 0x00, 0x4e, 0x13, 0xb5, 0xbf, 0x66, 0xc1, 0x85, 0xae, + 0x46, 0x28, 0x08, 0xc3, 0xd9, 0xc6, 0x4e, 0xe4, 0x2c, 0x86, 0xc4, 0x25, 0x7e, 0xec, 0x39, 0xcd, + 0x5a, 0x8b, 0xd4, 0x35, 0x05, 0x2b, 0xb3, 0xf5, 0xb9, 0xbe, 0x5a, 0x9b, 0xef, 0xc4, 0xc0, 0x39, + 0x35, 0xd1, 0x32, 0xa0, 0x4e, 0x88, 0x98, 0x61, 0x16, 0x77, 0xb1, 0x93, 0x1e, 0xce, 0xa8, 0xb1, + 0x70, 0xe5, 0x37, 0x7e, 0xf7, 0xe2, 0xc7, 0x7e, 0xf3, 0x77, 0x2f, 0x7e, 0xec, 0xb7, 0x7e, 0xf7, + 0xe2, 0xc7, 0xbe, 0xfb, 0xe0, 0xa2, 0xf5, 0x1b, 0x07, 0x17, 0xad, 0xdf, 0x3c, 0xb8, 0x68, 0xfd, + 0xd6, 0xc1, 0x45, 0xeb, 0x77, 0x0e, 0x2e, 0x5a, 0x5f, 0xf9, 0xbd, 0x8b, 0x1f, 0x7b, 0xab, 0xb0, + 0xfb, 0xfc, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x43, 0x82, 0xdf, 0x38, 0xd6, 0xec, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 9d23f83a561..9f0dfa02575 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -4863,5 +4863,16 @@ message WeightedPodAffinityTerm { // WindowsSecurityContextOptions contain Windows-specific options and credentials. message WindowsSecurityContextOptions { + // GMSACredentialSpecName is the name of the GMSA credential spec to use. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + optional string gmsaCredentialSpecName = 1; + + // GMSACredentialSpec is where the GMSA admission webhook + // (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + // GMSA credential spec named by the GMSACredentialSpecName field. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + optional string gmsaCredentialSpec = 2; } diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index d40e1033a96..2a0320d1a89 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -2410,18 +2410,22 @@ type PodConditionType string // These are valid conditions of pod. const ( - // PodScheduled represents status of the scheduling process for this pod. - PodScheduled PodConditionType = "PodScheduled" + // ContainersReady indicates whether all containers in the pod are ready. + ContainersReady PodConditionType = "ContainersReady" + // PodInitialized means that all init containers in the pod have started successfully. + PodInitialized PodConditionType = "Initialized" // PodReady means the pod is able to service requests and should be added to the // load balancing pools of all matching services. PodReady PodConditionType = "Ready" - // PodInitialized means that all init containers in the pod have started successfully. - PodInitialized PodConditionType = "Initialized" + // PodScheduled represents status of the scheduling process for this pod. + PodScheduled PodConditionType = "PodScheduled" +) + +// These are reasons for a pod's transition to a condition. +const ( // PodReasonUnschedulable reason in PodScheduled PodCondition means that the scheduler // can't schedule the pod right now, for example due to insufficient resources in the cluster. PodReasonUnschedulable = "Unschedulable" - // ContainersReady indicates whether all containers in the pod are ready. - ContainersReady PodConditionType = "ContainersReady" ) // PodCondition contains details for the current condition of this pod. @@ -5355,7 +5359,17 @@ type SELinuxOptions struct { // WindowsSecurityContextOptions contain Windows-specific options and credentials. type WindowsSecurityContextOptions struct { - // intentionally left empty for now + // GMSACredentialSpecName is the name of the GMSA credential spec to use. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + GMSACredentialSpecName *string `json:"gmsaCredentialSpecName,omitempty" protobuf:"bytes,1,opt,name=gmsaCredentialSpecName"` + + // GMSACredentialSpec is where the GMSA admission webhook + // (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + // GMSA credential spec named by the GMSACredentialSpecName field. + // This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag. + // +optional + GMSACredentialSpec *string `json:"gmsaCredentialSpec,omitempty" protobuf:"bytes,2,opt,name=gmsaCredentialSpec"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index b300453dfa2..85691d91f72 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -2363,7 +2363,9 @@ func (WeightedPodAffinityTerm) SwaggerDoc() map[string]string { } var map_WindowsSecurityContextOptions = map[string]string{ - "": "WindowsSecurityContextOptions contain Windows-specific options and credentials.", + "": "WindowsSecurityContextOptions contain Windows-specific options and credentials.", + "gmsaCredentialSpecName": "GMSACredentialSpecName is the name of the GMSA credential spec to use. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.", + "gmsaCredentialSpec": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. This field is alpha-level and is only honored by servers that enable the WindowsGMSA feature flag.", } func (WindowsSecurityContextOptions) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index 2d7741cca81..e434b4276f7 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -3457,7 +3457,7 @@ func (in *PodSecurityContext) DeepCopyInto(out *PodSecurityContext) { if in.WindowsOptions != nil { in, out := &in.WindowsOptions, &out.WindowsOptions *out = new(WindowsSecurityContextOptions) - **out = **in + (*in).DeepCopyInto(*out) } if in.RunAsUser != nil { in, out := &in.RunAsUser, &out.RunAsUser @@ -4656,7 +4656,7 @@ func (in *SecurityContext) DeepCopyInto(out *SecurityContext) { if in.WindowsOptions != nil { in, out := &in.WindowsOptions, &out.WindowsOptions *out = new(WindowsSecurityContextOptions) - **out = **in + (*in).DeepCopyInto(*out) } if in.RunAsUser != nil { in, out := &in.RunAsUser, &out.RunAsUser @@ -5490,6 +5490,16 @@ func (in *WeightedPodAffinityTerm) DeepCopy() *WeightedPodAffinityTerm { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WindowsSecurityContextOptions) DeepCopyInto(out *WindowsSecurityContextOptions) { *out = *in + if in.GMSACredentialSpecName != nil { + in, out := &in.GMSACredentialSpecName, &out.GMSACredentialSpecName + *out = new(string) + **out = **in + } + if in.GMSACredentialSpec != nil { + in, out := &in.GMSACredentialSpec, &out.GMSACredentialSpec + *out = new(string) + **out = **in + } return } diff --git a/staging/src/k8s.io/api/go.sum b/staging/src/k8s.io/api/go.sum index 77466c65fe6..0728d5a5246 100644 --- a/staging/src/k8s.io/api/go.sum +++ b/staging/src/k8s.io/api/go.sum @@ -50,8 +50,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.mod b/staging/src/k8s.io/apiextensions-apiserver/go.mod index 1072d701460..d8031342548 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.mod +++ b/staging/src/k8s.io/apiextensions-apiserver/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-openapi/strfmt v0.17.0 github.com/go-openapi/validate v0.18.0 github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 + github.com/google/go-cmp v0.3.0 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -31,7 +32,7 @@ require ( k8s.io/client-go v0.0.0 k8s.io/code-generator v0.0.0 k8s.io/component-base v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 k8s.io/utils v0.0.0-20190221042446-c2654d5206da sigs.k8s.io/yaml v1.1.0 diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index de2364aeb5d..f89ac484688 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -232,8 +232,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go index 03344e2d025..8daa1f960ff 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go @@ -393,8 +393,11 @@ type CustomResourceSubresourceScale struct { StatusReplicasPath string // LabelSelectorPath defines the JSON path inside of a CustomResource that corresponds to Scale.Status.Selector. // Only JSON paths without the array notation are allowed. - // Must be a JSON Path under .status. + // Must be a JSON Path under .status or .spec. // Must be set to work with HPA. + // The field pointed by this JSON path must be a string field (not a complex selector struct) + // which contains a serialized label selector in string form. + // More info: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions#scale-subresource // If there is no value under the given path in the CustomResource, the status label selector value in the /scale // subresource will default to the empty string. // +optional diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto index 166ad8bd135..7aa8c8b2042 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto @@ -327,8 +327,11 @@ message CustomResourceSubresourceScale { // LabelSelectorPath defines the JSON path inside of a CustomResource that corresponds to Scale.Status.Selector. // Only JSON paths without the array notation are allowed. - // Must be a JSON Path under .status. + // Must be a JSON Path under .status or .spec. // Must be set to work with HPA. + // The field pointed by this JSON path must be a string field (not a complex selector struct) + // which contains a serialized label selector in string form. + // More info: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions#scale-subresource // If there is no value under the given path in the CustomResource, the status label selector value in the /scale // subresource will default to the empty string. // +optional diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go index 32d6ee7e8e1..715d107e97c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go @@ -412,8 +412,11 @@ type CustomResourceSubresourceScale struct { StatusReplicasPath string `json:"statusReplicasPath" protobuf:"bytes,2,opt,name=statusReplicasPath"` // LabelSelectorPath defines the JSON path inside of a CustomResource that corresponds to Scale.Status.Selector. // Only JSON paths without the array notation are allowed. - // Must be a JSON Path under .status. + // Must be a JSON Path under .status or .spec. // Must be set to work with HPA. + // The field pointed by this JSON path must be a string field (not a complex selector struct) + // which contains a serialized label selector in string form. + // More info: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions#scale-subresource // If there is no value under the given path in the CustomResource, the status label selector value in the /scale // subresource will default to the empty string. // +optional diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 59cbedaac83..1986930d6b1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -794,8 +794,8 @@ func ValidateCustomResourceDefinitionSubresources(subresources *apiextensions.Cu if subresources.Scale.LabelSelectorPath != nil && len(*subresources.Scale.LabelSelectorPath) > 0 { if errs := validateSimpleJSONPath(*subresources.Scale.LabelSelectorPath, fldPath.Child("scale.labelSelectorPath")); len(errs) > 0 { allErrs = append(allErrs, errs...) - } else if !strings.HasPrefix(*subresources.Scale.LabelSelectorPath, ".status.") { - allErrs = append(allErrs, field.Invalid(fldPath.Child("scale.labelSelectorPath"), subresources.Scale.LabelSelectorPath, "should be a json path under .status")) + } else if !strings.HasPrefix(*subresources.Scale.LabelSelectorPath, ".spec.") && !strings.HasPrefix(*subresources.Scale.LabelSelectorPath, ".status.") { + allErrs = append(allErrs, field.Invalid(fldPath.Child("scale.labelSelectorPath"), subresources.Scale.LabelSelectorPath, "should be a json path under either .spec or .status")) } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go index 6cf96dcc575..14bc3be9f96 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go @@ -1162,6 +1162,83 @@ func TestValidateCustomResourceDefinition(t *testing.T) { required("spec", "versions[1]", "schema", "openAPIV3Schema"), }, }, + { + name: "labelSelectorPath outside of .spec and .status", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version0", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + // null labelSelectorPath + Name: "version0", + Served: true, + Storage: true, + Subresources: &apiextensions.CustomResourceSubresources{ + Scale: &apiextensions.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + }, + }, + }, + { + // labelSelectorPath under .status + Name: "version1", + Served: true, + Storage: false, + Subresources: &apiextensions.CustomResourceSubresources{ + Scale: &apiextensions.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: strPtr(".status.labelSelector"), + }, + }, + }, + { + // labelSelectorPath under .spec + Name: "version2", + Served: true, + Storage: false, + Subresources: &apiextensions.CustomResourceSubresources{ + Scale: &apiextensions.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: strPtr(".spec.labelSelector"), + }, + }, + }, + { + // labelSelectorPath outside of .spec and .status + Name: "version3", + Served: true, + Storage: false, + Subresources: &apiextensions.CustomResourceSubresources{ + Scale: &apiextensions.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + LabelSelectorPath: strPtr(".labelSelector"), + }, + }, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(true), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version0"}, + }, + }, + errors: []validationMatch{ + invalid("spec", "versions[3]", "subresources", "scale", "labelSelectorPath"), + }, + }, } for _, tc := range tests { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 9883471c647..e7b3713739f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -213,7 +213,11 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return nil }) s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error { - if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) { + // OpenAPIVersionedService and StaticOpenAPISpec are populated in generic apiserver PrepareRun(). + // Together they serve the /openapi/v2 endpoint on a generic apiserver. A generic apiserver may + // choose to not enable OpenAPI by having null openAPIConfig, and thus OpenAPIVersionedService + // and StaticOpenAPISpec are both null. In that case we don't run the CRD OpenAPI controller. + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) && s.GenericAPIServer.OpenAPIVersionedService != nil && s.GenericAPIServer.StaticOpenAPISpec != nil { go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go index 3ad56720340..d9827ca4cb8 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go @@ -35,14 +35,15 @@ type CRConverterFactory struct { // webhookConverterFactory is the factory for webhook converters. // This field should not be used if CustomResourceWebhookConversion feature is disabled. webhookConverterFactory *webhookConverterFactory - converterMetricFactory *converterMetricFactory } +// converterMetricFactorySingleton protects us from reregistration of metrics on repeated +// apiextensions-apiserver runs. +var converterMetricFactorySingleton = newConverterMertricFactory() + // NewCRConverterFactory creates a new CRConverterFactory func NewCRConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*CRConverterFactory, error) { - converterFactory := &CRConverterFactory{ - converterMetricFactory: newConverterMertricFactory(), - } + converterFactory := &CRConverterFactory{} if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { webhookConverterFactory, err := newWebhookConverterFactory(serviceResolver, authResolverWrapper) if err != nil { @@ -72,7 +73,7 @@ func (m *CRConverterFactory) NewConverter(crd *apiextensions.CustomResourceDefin if err != nil { return nil, nil, err } - converter, err = m.converterMetricFactory.addMetrics("webhook", crd.Name, converter) + converter, err = converterMetricFactorySingleton.addMetrics("webhook", crd.Name, converter) if err != nil { return nil, nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index c9088ad4962..b977de65210 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -493,6 +493,23 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource statusScopes := map[string]*handlers.RequestScope{} scaleScopes := map[string]*handlers.RequestScope{} + structuralSchemas := map[string]*structuralschema.Structural{} + for _, v := range crd.Spec.Versions { + val, err := apiextensions.GetSchemaForVersion(crd, v.Name) + if err != nil { + utilruntime.HandleError(err) + return nil, fmt.Errorf("the server could not properly serve the CR schema") + } + if val == nil { + continue + } + structuralSchemas[v.Name], err = structuralschema.NewStructural(val.OpenAPIV3Schema) + if *crd.Spec.PreserveUnknownFields == false && err != nil { + utilruntime.HandleError(err) + return nil, fmt.Errorf("the server could not properly serve the CR schema") // validation should avoid this + } + } + for _, v := range crd.Spec.Versions { safeConverter, unsafeConverter, err := r.converterFactory.NewConverter(crd) if err != nil { @@ -529,14 +546,6 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource return nil, fmt.Errorf("unexpected nil spec.preserveUnknownFields in the CustomResourceDefinition") } - var structuralSchema *structuralschema.Structural - if validationSchema != nil { - structuralSchema, err = structuralschema.NewStructural(validationSchema.OpenAPIV3Schema) - if *crd.Spec.PreserveUnknownFields == false && err != nil { - return nil, err // validation should avoid this - } - } - var statusSpec *apiextensions.CustomResourceSubresourceStatus var statusValidator *validate.SchemaValidator subresources, err := apiextensions.GetSubresourcesForVersion(crd, v.Name) @@ -591,7 +600,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource converter: safeConverter, decoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name}, encoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: storageVersion}, - structuralSchema: structuralSchema, + structuralSchemas: structuralSchemas, structuralSchemaGK: kind.GroupKind(), preserveUnknownFields: *crd.Spec.PreserveUnknownFields, }, @@ -619,7 +628,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource typer: typer, creator: creator, converter: safeConverter, - structuralSchema: structuralSchema, + structuralSchemas: structuralSchemas, structuralSchemaGK: kind.GroupKind(), preserveUnknownFields: *crd.Spec.PreserveUnknownFields, }, @@ -676,7 +685,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource statusScope.Serializer = unstructuredNegotiatedSerializer{ typer: typer, creator: creator, converter: safeConverter, - structuralSchema: structuralSchema, + structuralSchemas: structuralSchemas, structuralSchemaGK: kind.GroupKind(), preserveUnknownFields: *crd.Spec.PreserveUnknownFields, } @@ -715,7 +724,7 @@ type unstructuredNegotiatedSerializer struct { creator runtime.ObjectCreater converter runtime.ObjectConvertor - structuralSchema *structuralschema.Structural + structuralSchemas map[string]*structuralschema.Structural // by version structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -750,7 +759,7 @@ func (s unstructuredNegotiatedSerializer) EncoderForVersion(encoder runtime.Enco } func (s unstructuredNegotiatedSerializer) DecoderToVersion(decoder runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder { - d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{structuralSchema: s.structuralSchema, structuralSchemaGK: s.structuralSchemaGK, preserveUnknownFields: s.preserveUnknownFields}} + d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{structuralSchemas: s.structuralSchemas, structuralSchemaGK: s.structuralSchemaGK, preserveUnknownFields: s.preserveUnknownFields}} return versioning.NewDefaultingCodecForScheme(Scheme, nil, d, nil, gv) } @@ -842,7 +851,7 @@ type crdConversionRESTOptionsGetter struct { converter runtime.ObjectConvertor encoderVersion schema.GroupVersion decoderVersion schema.GroupVersion - structuralSchema *structuralschema.Structural + structuralSchemas map[string]*structuralschema.Structural // by version structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -853,12 +862,12 @@ func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupReso d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{ // drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation) dropInvalidMetadata: true, - structuralSchema: t.structuralSchema, + structuralSchemas: t.structuralSchemas, structuralSchemaGK: t.structuralSchemaGK, preserveUnknownFields: t.preserveUnknownFields, }} c := schemaCoercingConverter{delegate: t.converter, validator: unstructuredSchemaCoercer{ - structuralSchema: t.structuralSchema, + structuralSchemas: t.structuralSchemas, structuralSchemaGK: t.structuralSchemaGK, preserveUnknownFields: t.preserveUnknownFields, }} @@ -950,7 +959,7 @@ func (v schemaCoercingConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, type unstructuredSchemaCoercer struct { dropInvalidMetadata bool - structuralSchema *structuralschema.Structural + structuralSchemas map[string]*structuralschema.Structural structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -976,7 +985,7 @@ func (v *unstructuredSchemaCoercer) apply(u *unstructured.Unstructured) error { return err } if !v.preserveUnknownFields && gv.Group == v.structuralSchemaGK.Group && kind == v.structuralSchemaGK.Kind { - structuralpruning.Prune(u.Object, v.structuralSchema) + structuralpruning.Prune(u.Object, v.structuralSchemas[gv.Version]) } // restore meta fields, starting clean diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD index 426c8bd6fb1..6717f67507f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD @@ -7,6 +7,7 @@ go_library( "convert.go", "goopenapi.go", "structural.go", + "unfold.go", "validation.go", "visitor.go", "zz_generated.deepcopy.go", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/unfold.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/unfold.go new file mode 100644 index 00000000000..d135757ee7b --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/unfold.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +// Unfold expands vendor extensions of a structural schema. +// It mutates the receiver. +func (s *Structural) Unfold() *Structural { + if s == nil { + return nil + } + + mapper := Visitor{ + Structural: func(s *Structural) bool { + if !s.XIntOrString { + return false + } + + skipAnyOf := isIntOrStringAnyOfPattern(s) + skipFirstAllOfAnyOf := isIntOrStringAllOfPattern(s) + if skipAnyOf || skipFirstAllOfAnyOf { + return false + } + + if s.AnyOf == nil { + s.AnyOf = []NestedValueValidation{ + {ForbiddenGenerics: Generic{Type: "integer"}}, + {ForbiddenGenerics: Generic{Type: "string"}}, + } + } else { + s.AllOf = append([]NestedValueValidation{ + { + ValueValidation: ValueValidation{ + AnyOf: []NestedValueValidation{ + {ForbiddenGenerics: Generic{Type: "integer"}}, + {ForbiddenGenerics: Generic{Type: "string"}}, + }, + }, + }, + }, s.AllOf...) + } + + return true + }, + NestedValueValidation: nil, // x-kubernetes-int-or-string cannot be set in nested value validation + } + mapper.Visit(s) + + return s +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/validation.go index a6da5b1c899..e864fe8c249 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/validation.go @@ -97,15 +97,8 @@ func validateStructuralInvariants(s *Structural, lvl level, fldPath *field.Path) // - type: integer // - type: string // - ... zero or more - skipAnyOf := false - skipFirstAllOfAnyOf := false - if s.XIntOrString && s.ValueValidation != nil { - if len(s.ValueValidation.AnyOf) == 2 && reflect.DeepEqual(s.ValueValidation.AnyOf, intOrStringAnyOf) { - skipAnyOf = true - } else if len(s.ValueValidation.AllOf) >= 1 && len(s.ValueValidation.AllOf[0].AnyOf) == 2 && reflect.DeepEqual(s.ValueValidation.AllOf[0].AnyOf, intOrStringAnyOf) { - skipFirstAllOfAnyOf = true - } - } + skipAnyOf := isIntOrStringAnyOfPattern(s) + skipFirstAllOfAnyOf := isIntOrStringAllOfPattern(s) allErrs = append(allErrs, validateValueValidation(s.ValueValidation, skipAnyOf, skipFirstAllOfAnyOf, lvl, fldPath)...) @@ -157,6 +150,20 @@ func validateStructuralInvariants(s *Structural, lvl level, fldPath *field.Path) return allErrs } +func isIntOrStringAnyOfPattern(s *Structural) bool { + if s == nil || s.ValueValidation == nil { + return false + } + return len(s.ValueValidation.AnyOf) == 2 && reflect.DeepEqual(s.ValueValidation.AnyOf, intOrStringAnyOf) +} + +func isIntOrStringAllOfPattern(s *Structural) bool { + if s == nil || s.ValueValidation == nil { + return false + } + return len(s.ValueValidation.AllOf) >= 1 && len(s.ValueValidation.AllOf[0].AnyOf) == 2 && reflect.DeepEqual(s.ValueValidation.AllOf[0].AnyOf, intOrStringAnyOf) +} + // validateGeneric checks the generic fields of a structural schema. func validateGeneric(g *Generic, lvl level, fldPath *field.Path) field.ErrorList { if g == nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/nonstructuralschema/nonstructuralschema_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/nonstructuralschema/nonstructuralschema_controller.go index e12bd20bfd1..98c4002bdd5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/nonstructuralschema/nonstructuralschema_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/nonstructuralschema/nonstructuralschema_controller.go @@ -18,6 +18,7 @@ package nonstructuralschema import ( "fmt" + "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -47,6 +48,11 @@ type ConditionController struct { syncFn func(key string) error queue workqueue.RateLimitingInterface + + // last generation this controller updated the condition per CRD name (to avoid two + // different version of the apiextensions-apiservers in HA to fight for the right message) + lastSeenGenerationLock sync.Mutex + lastSeenGeneration map[string]int64 } // NewConditionController constructs a non-structural schema condition controller. @@ -55,16 +61,17 @@ func NewConditionController( crdClient client.CustomResourceDefinitionsGetter, ) *ConditionController { c := &ConditionController{ - crdClient: crdClient, - crdLister: crdInformer.Lister(), - crdSynced: crdInformer.Informer().HasSynced, - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "non_structural_schema_condition_controller"), + crdClient: crdClient, + crdLister: crdInformer.Lister(), + crdSynced: crdInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "non_structural_schema_condition_controller"), + lastSeenGeneration: map[string]int64{}, } crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addCustomResourceDefinition, UpdateFunc: c.updateCustomResourceDefinition, - DeleteFunc: nil, + DeleteFunc: c.deleteCustomResourceDefinition, }) c.syncFn = c.sync @@ -130,6 +137,14 @@ func (c *ConditionController) sync(key string) error { return err } + // avoid repeated calculation for the same generation + c.lastSeenGenerationLock.Lock() + lastSeen, seenBefore := c.lastSeenGeneration[inCustomResourceDefinition.Name] + c.lastSeenGenerationLock.Unlock() + if seenBefore && inCustomResourceDefinition.Generation <= lastSeen { + return nil + } + // check old condition cond := calculateCondition(inCustomResourceDefinition) old := apiextensions.FindCRDCondition(inCustomResourceDefinition, apiextensions.NonStructuralSchema) @@ -159,6 +174,12 @@ func (c *ConditionController) sync(key string) error { return err } + // store generation in order to avoid repeated updates for the same generation (and potential + // fights of API server in HA environments). + c.lastSeenGenerationLock.Lock() + defer c.lastSeenGenerationLock.Unlock() + c.lastSeenGeneration[crd.Name] = crd.Generation + return nil } @@ -227,3 +248,23 @@ func (c *ConditionController) updateCustomResourceDefinition(obj, _ interface{}) klog.V(4).Infof("Updating %s", castObj.Name) c.enqueue(castObj) } + +func (c *ConditionController) deleteCustomResourceDefinition(obj interface{}) { + castObj, ok := obj.(*apiextensions.CustomResourceDefinition) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Couldn't get object from tombstone %#v", obj) + return + } + castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition) + if !ok { + klog.Errorf("Tombstone contained object that is not expected %#v", obj) + return + } + } + + c.lastSeenGenerationLock.Lock() + defer c.lastSeenGenerationLock.Unlock() + delete(c.lastSeenGeneration, castObj.Name) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD index 8f750597808..c534778ae01 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD @@ -14,7 +14,7 @@ go_library( deps = [ "//staging/src/k8s.io/api/autoscaling/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:go_default_library", @@ -49,10 +49,12 @@ go_test( deps = [ "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/github.com/googleapis/gnostic/compiler:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go index 2bbffda1742..ee31526cda5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder.go @@ -26,6 +26,7 @@ import ( "github.com/go-openapi/spec" v1 "k8s.io/api/autoscaling/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1" "k8s.io/apimachinery/pkg/runtime" @@ -58,22 +59,24 @@ var namer *openapi.DefinitionNamer // BuildSwagger builds swagger for the given crd in the given version func BuildSwagger(crd *apiextensions.CustomResourceDefinition, version string) (*spec.Swagger, error) { - var schema *spec.Schema + var schema *structuralschema.Structural s, err := apiextensions.GetSchemaForVersion(crd, version) if err != nil { return nil, err } if s != nil && s.OpenAPIV3Schema != nil { - schema, err = ConvertJSONSchemaPropsToOpenAPIv2Schema(s.OpenAPIV3Schema) - if err != nil { - return nil, err + ss, err := structuralschema.NewStructural(s.OpenAPIV3Schema) + if err == nil && len(structuralschema.ValidateStructural(ss, nil)) == 0 { + // skip non-structural schemas + schema = ss.Unfold() } } + // TODO(roycaihw): remove the WebService templating below. The following logic // comes from function registerResourceHandlers() in k8s.io/apiserver. // Alternatives are either (ideally) refactoring registerResourceHandlers() to // reuse the code, or faking an APIInstaller for CR to feed to registerResourceHandlers(). - b := newBuilder(crd, version, schema) + b := newBuilder(crd, version, schema, true) // Sample response types for building web service sample := &CRDCanonicalTypeNamer{ @@ -288,23 +291,27 @@ func (b *builder) buildRoute(root, path, action, verb string, sample interface{} // buildKubeNative builds input schema with Kubernetes' native object meta, type meta and // extensions -func (b *builder) buildKubeNative(schema *spec.Schema) *spec.Schema { +func (b *builder) buildKubeNative(schema *structuralschema.Structural, v2 bool) (ret *spec.Schema) { // only add properties if we have a schema. Otherwise, kubectl would (wrongly) assume additionalProperties=false // and forbid anything outside of apiVersion, kind and metadata. We have to fix kubectl to stop doing this, e.g. by // adding additionalProperties=true support to explicitly allow additional fields. // TODO: fix kubectl to understand additionalProperties=true if schema == nil { - schema = &spec.Schema{ + ret = &spec.Schema{ SchemaProps: spec.SchemaProps{Type: []string{"object"}}, } // no, we cannot add more properties here, not even TypeMeta/ObjectMeta because kubectl will complain about // unknown fields for anything else. } else { - schema.SetProperty("metadata", *spec.RefSchema(objectMetaSchemaRef). + if v2 { + schema = ToStructuralOpenAPIV2(schema) + } + ret = schema.ToGoOpenAPI() + ret.SetProperty("metadata", *spec.RefSchema(objectMetaSchemaRef). WithDescription(swaggerPartialObjectMetadataDescriptions["metadata"])) - addTypeMetaProperties(schema) + addTypeMetaProperties(ret) } - schema.AddExtension(endpoints.ROUTE_META_GVK, []interface{}{ + ret.AddExtension(endpoints.ROUTE_META_GVK, []interface{}{ map[string]interface{}{ "group": b.group, "version": b.version, @@ -312,7 +319,7 @@ func (b *builder) buildKubeNative(schema *spec.Schema) *spec.Schema { }, }) - return schema + return ret } // getDefinition gets definition for given Kubernetes type. This function is extracted from @@ -391,7 +398,7 @@ func (b *builder) getOpenAPIConfig() *common.Config { } } -func newBuilder(crd *apiextensions.CustomResourceDefinition, version string, schema *spec.Schema) *builder { +func newBuilder(crd *apiextensions.CustomResourceDefinition, version string, schema *structuralschema.Structural, v2 bool) *builder { b := &builder{ schema: &spec.Schema{ SchemaProps: spec.SchemaProps{Type: []string{"object"}}, @@ -410,7 +417,7 @@ func newBuilder(crd *apiextensions.CustomResourceDefinition, version string, sch } // Pre-build schema with Kubernetes native properties - b.schema = b.buildKubeNative(schema) + b.schema = b.buildKubeNative(schema, v2) b.listSchema = b.buildListSchema() return b diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go index 2738137aa5b..3b175cf282e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go @@ -22,14 +22,14 @@ import ( "github.com/go-openapi/spec" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" ) func TestNewBuilder(t *testing.T) { - type args struct { - } tests := []struct { name string @@ -37,41 +37,302 @@ func TestNewBuilder(t *testing.T) { wantedSchema string wantedItemsSchema string + + v2 bool // produce OpenAPIv2 }{ { "nil", "", `{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + true, }, - {"empty", - "{}", - `{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + {"with properties", + `{"type":"object","properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, + `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + true, }, - {"empty properties", - `{"properties":{"spec":{},"status":{}}}`, - `{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{},"status":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, - }, - {"filled properties", - `{"properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`, - `{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, - `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, - }, - {"type", + {"type only", `{"type":"object"}`, - `{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, + `{"type":"object","properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"metadata":{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + true, + }, + {"with extensions", + ` +{ + "type":"object", + "properties": { + "int-or-string-1": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, + "int-or-string-2": { + "x-kubernetes-int-or-string": true, + "allOf": [{ + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, { + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-3": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ], + "allOf": [{ + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-4": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"minimum": 42.0} + ] + }, + "int-or-string-5": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"minimum": 42.0} + ], + "allOf": [ + {"minimum": 42.0} + ] + }, + "int-or-string-6": { + "x-kubernetes-int-or-string": true + }, + "preserve-unknown-fields": { + "x-kubernetes-preserve-unknown-fields": true + }, + "embedded-object": { + "x-kubernetes-embedded-resource": true, + "x-kubernetes-preserve-unknown-fields": true, + "type": "object" + } + } +}`, + ` +{ + "type":"object", + "properties": { + "apiVersion": {"type":"string"}, + "kind": {"type":"string"}, + "metadata": {"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}, + "int-or-string-1": { + "x-kubernetes-int-or-string": true + }, + "int-or-string-2": { + "x-kubernetes-int-or-string": true + }, + "int-or-string-3": { + "x-kubernetes-int-or-string": true + }, + "int-or-string-4": { + "x-kubernetes-int-or-string": true + }, + "int-or-string-5": { + "x-kubernetes-int-or-string": true + }, + "int-or-string-6": { + "x-kubernetes-int-or-string": true + }, + "preserve-unknown-fields": { + "x-kubernetes-preserve-unknown-fields": true + }, + "embedded-object": { + "x-kubernetes-embedded-resource": true, + "x-kubernetes-preserve-unknown-fields": true, + "type": "object" + } + }, + "x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}] +}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + true, + }, + {"with extensions as v3 schema", + ` +{ + "type":"object", + "properties": { + "int-or-string-1": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, + "int-or-string-2": { + "x-kubernetes-int-or-string": true, + "allOf": [{ + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, { + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-3": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ], + "allOf": [{ + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-4": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"minimum": 42.0} + ] + }, + "int-or-string-5": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"minimum": 42.0} + ], + "allOf": [ + {"minimum": 42.0} + ] + }, + "int-or-string-6": { + "x-kubernetes-int-or-string": true + }, + "preserve-unknown-fields": { + "x-kubernetes-preserve-unknown-fields": true + }, + "embedded-object": { + "x-kubernetes-embedded-resource": true, + "x-kubernetes-preserve-unknown-fields": true, + "type": "object" + } + } +}`, + ` +{ + "type":"object", + "properties": { + "apiVersion": {"type":"string"}, + "kind": {"type":"string"}, + "metadata": {"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}, + "int-or-string-1": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, + "int-or-string-2": { + "x-kubernetes-int-or-string": true, + "allOf": [{ + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, { + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-3": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ], + "allOf": [{ + "anyOf": [ + {"minimum": 42.0} + ] + }] + }, + "int-or-string-4": { + "x-kubernetes-int-or-string": true, + "allOf": [{ + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }], + "anyOf": [ + {"minimum": 42.0} + ] + }, + "int-or-string-5": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"minimum": 42.0} + ], + "allOf": [{ + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, { + "minimum": 42.0 + }] + }, + "int-or-string-6": { + "x-kubernetes-int-or-string": true, + "anyOf": [ + {"type":"integer"}, + {"type":"string"} + ] + }, + "preserve-unknown-fields": { + "x-kubernetes-preserve-unknown-fields": true + }, + "embedded-object": { + "x-kubernetes-embedded-resource": true, + "x-kubernetes-preserve-unknown-fields": true, + "type": "object" + } + }, + "x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}] +}`, + `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`, + false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var schema *spec.Schema + var schema *structuralschema.Structural if len(tt.schema) > 0 { - schema = &spec.Schema{} - if err := json.Unmarshal([]byte(tt.schema), schema); err != nil { + v1beta1Schema := &v1beta1.JSONSchemaProps{} + if err := json.Unmarshal([]byte(tt.schema), &v1beta1Schema); err != nil { t.Fatal(err) } + internalSchema := &apiextensions.JSONSchemaProps{} + v1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(v1beta1Schema, internalSchema, nil) + var err error + schema, err = structuralschema.NewStructural(internalSchema) + if err != nil { + t.Fatalf("structural schema error: %v", err) + } + if errs := structuralschema.ValidateStructural(schema, nil); len(errs) > 0 { + t.Fatalf("structural schema validation error: %v", errs.ToAggregate()) + } + schema = schema.Unfold() } got := newBuilder(&apiextensions.CustomResourceDefinition{ @@ -86,7 +347,7 @@ func TestNewBuilder(t *testing.T) { }, Scope: apiextensions.NamespaceScoped, }, - }, "v1", schema) + }, "v1", schema, tt.v2) var wantedSchema, wantedItemsSchema spec.Schema if err := json.Unmarshal([]byte(tt.wantedSchema), &wantedSchema); err != nil { @@ -103,14 +364,12 @@ func TestNewBuilder(t *testing.T) { } // wipe out TypeMeta/ObjectMeta content, with those many lines of descriptions. We trust that they match here. - if _, found := got.schema.Properties["kind"]; found { - got.schema.Properties["kind"] = spec.Schema{} - } - if _, found := got.schema.Properties["apiVersion"]; found { - got.schema.Properties["apiVersion"] = spec.Schema{} - } - if _, found := got.schema.Properties["metadata"]; found { - got.schema.Properties["metadata"] = spec.Schema{} + for _, metaField := range []string{"kind", "apiVersion", "metadata"} { + if _, found := got.schema.Properties["kind"]; found { + prop := got.schema.Properties[metaField] + prop.Description = "" + got.schema.Properties[metaField] = prop + } } if !reflect.DeepEqual(&wantedSchema, got.schema) { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion.go index 743fa196ac4..55360ea5303 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion.go @@ -17,106 +17,60 @@ limitations under the License. package openapi import ( - "strings" - - "github.com/go-openapi/spec" - - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" ) -// ConvertJSONSchemaPropsToOpenAPIv2Schema converts our internal OpenAPI v3 schema -// (*apiextensions.JSONSchemaProps) to an OpenAPI v2 schema (*spec.Schema). -func ConvertJSONSchemaPropsToOpenAPIv2Schema(in *apiextensions.JSONSchemaProps) (*spec.Schema, error) { +// ToStructuralOpenAPIV2 converts our internal OpenAPI v3 structural schema to +// to a v2 compatible schema. +func ToStructuralOpenAPIV2(in *structuralschema.Structural) *structuralschema.Structural { if in == nil { - return nil, nil + return nil } - // dirty hack to temporarily set the type at the root. See continuation at the func bottom. - // TODO: remove for Kubernetes 1.15 - oldRootType := in.Type - if len(in.Type) == 0 { - in.Type = "object" - } + out := in.DeepCopy() // Remove unsupported fields in OpenAPI v2 recursively - out := new(spec.Schema) - validation.ConvertJSONSchemaPropsWithPostProcess(in, out, func(p *spec.Schema) error { - p.OneOf = nil - // TODO(roycaihw): preserve cases where we only have one subtree in AnyOf, same for OneOf - p.AnyOf = nil - p.Not = nil - - // TODO: drop everything below in 1.15 when we have passed one version skew towards kube-openapi in <1.14, which rejects valid openapi schemata - - if p.Ref.String() != "" { - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R95 - p.Properties = nil - - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R99 - p.Type = nil - - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R104 - if !strings.HasPrefix(p.Ref.String(), "#/definitions/") { - p.Ref = spec.Ref{} - } - } - - switch { - case len(p.Type) == 2 && (p.Type[0] == "null" || p.Type[1] == "null"): - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-ce77fea74b9dd098045004410023e0c3R219 - p.Type = nil - case len(p.Type) == 1: - switch p.Type[0] { - case "null": - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-ce77fea74b9dd098045004410023e0c3R219 - p.Type = nil - case "array": - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R183 - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R184 - if p.Items == nil || (p.Items.Schema == nil && len(p.Items.Schemas) != 1) { - p.Type = nil - p.Items = nil + mapper := structuralschema.Visitor{ + Structural: func(s *structuralschema.Structural) bool { + changed := false + if s.ValueValidation != nil { + if s.ValueValidation.AllOf != nil { + s.ValueValidation.AllOf = nil + changed = true + } + if s.ValueValidation.OneOf != nil { + s.ValueValidation.OneOf = nil + changed = true + } + if s.ValueValidation.AnyOf != nil { + s.ValueValidation.AnyOf = nil + changed = true + } + if s.ValueValidation.Not != nil { + s.ValueValidation.Not = nil + changed = true } } - case len(p.Type) > 1: - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R272 - // We also set Properties to null to enforce parseArbitrary at https://github.com/kubernetes/kube-openapi/blob/814a8073653e40e0e324205d093770d4e7bb811f/pkg/util/proto/document.go#L247 - p.Type = nil - p.Properties = nil - default: - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R248 - p.Properties = nil - } - // normalize items - if p.Items != nil && len(p.Items.Schemas) == 1 { - p.Items = &spec.SchemaOrArray{Schema: &p.Items.Schemas[0]} - } + // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-ce77fea74b9dd098045004410023e0c3R219 + if s.Nullable { + s.Type = "" + s.Nullable = false - // general fixups not supported by gnostic - p.ID = "" - p.Schema = "" - p.Definitions = nil - p.AdditionalItems = nil - p.Dependencies = nil - p.PatternProperties = nil - if p.ExternalDocs != nil && len(p.ExternalDocs.URL) == 0 { - p.ExternalDocs = nil - } - if p.Items != nil && p.Items.Schemas != nil { - p.Items = nil - } + // untyped values break if items or properties are set in kubectl + // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R183 + s.Items = nil + s.Properties = nil - return nil - }) + changed = true + } - // restore root level type in input, and remove it in output if we had added it - // TODO: remove with Kubernetes 1.15 - in.Type = oldRootType - if len(oldRootType) == 0 { - out.Type = nil + return changed + }, + // we drop all junctors above, and hence, never reach nested value validations + NestedValueValidation: nil, } + mapper.Visit(out) - return out, nil + return out } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion_test.go index a6c6fd94540..f88e070a010 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/conversion_test.go @@ -26,13 +26,15 @@ import ( "time" "github.com/go-openapi/spec" - "github.com/google/gofuzz" - "github.com/googleapis/gnostic/OpenAPIv2" + "github.com/google/go-cmp/cmp" + fuzz "github.com/google/gofuzz" + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" "github.com/googleapis/gnostic/compiler" "gopkg.in/yaml.v2" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/util/diff" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/kube-openapi/pkg/util/proto" ) @@ -94,11 +96,14 @@ properties: t.Fatal(err) } - schema, err := ConvertJSONSchemaPropsToOpenAPIv2Schema(&specInternal) + ss, err := structuralschema.NewStructural(&specInternal) if err != nil { t.Fatal(err) } + ssV2 := ToStructuralOpenAPIV2(ss) + schema := ssV2.ToGoOpenAPI() + if _, found := schema.Properties["spec"]; !found { t.Errorf("spec not found") } @@ -107,7 +112,7 @@ properties: } } -func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { +func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaByType(t *testing.T) { testStr := "test" testStr2 := "test2" testFloat64 := float64(6.4) @@ -115,41 +120,32 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { testApiextensionsJSON := apiextensions.JSON(testStr) tests := []struct { - name string - in *apiextensions.JSONSchemaProps - expected *spec.Schema + name string + in *apiextensions.JSONSchemaProps + expected *spec.Schema + expectError bool + expectDiff bool }{ { name: "id", in: &apiextensions.JSONSchemaProps{ ID: testStr, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: new(spec.Schema). - // WithID(testStr), + expectError: true, // rejected by kube validation and NewStructural }, { name: "$schema", in: &apiextensions.JSONSchemaProps{ Schema: "test", }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // Schema: "test", - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "$ref", in: &apiextensions.JSONSchemaProps{ Ref: &testStr, }, - expected: new(spec.Schema), - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R104 - // expected: spec.RefSchema(testStr), + expectError: true, // rejected by kube validation and NewStructural }, { name: "description", @@ -168,6 +164,14 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { expected: new(spec.Schema). Typed(testStr, testStr2), }, + { + name: "nullable", + in: &apiextensions.JSONSchemaProps{ + Type: "object", + Nullable: true, + }, + expected: new(spec.Schema), + }, { name: "title", in: &apiextensions.JSONSchemaProps{ @@ -317,18 +321,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, }, }, - expected: new(spec.Schema), - // https://github.com/kubernetes/kube-openapi/pull/143/files#diff-62afddb578e9db18fb32ffb6b7802d92R272 - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // Items: &spec.SchemaOrArray{ - // Schemas: []spec.Schema{ - // *spec.BooleanProperty(), - // *spec.StringProperty(), - // }, - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "allOf", @@ -338,8 +331,10 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { {Type: "string"}, }, }, - expected: new(spec.Schema). - WithAllOf(*spec.BooleanProperty(), *spec.StringProperty()), + expected: new(spec.Schema), + // intentionally not exported in v2 + // expected: new(spec.Schema). + // WithAllOf(*spec.BooleanProperty(), *spec.StringProperty()), }, { name: "oneOf", @@ -471,8 +466,10 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, }, }, - expected: new(spec.Schema). - WithAllOf(spec.Schema{}, spec.Schema{}, spec.Schema{}, *spec.StringProperty()), + expected: new(spec.Schema), + // not supported by OpenAPI v2 + allOf intentionally not exported + // expected: new(spec.Schema). + // WithAllOf(spec.Schema{}, spec.Schema{}, spec.Schema{}, *spec.StringProperty()), }, { name: "properties", @@ -485,22 +482,39 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { SetProperty(testStr, *spec.BooleanProperty()), }, { - name: "additionalProperties", + name: "additionalProperties schema", in: &apiextensions.JSONSchemaProps{ AdditionalProperties: &apiextensions.JSONSchemaPropsOrBool{ - Allows: true, + Allows: false, Schema: &apiextensions.JSONSchemaProps{Type: "boolean"}, }, }, expected: &spec.Schema{ SchemaProps: spec.SchemaProps{ AdditionalProperties: &spec.SchemaOrBool{ - Allows: true, + Allows: false, Schema: spec.BooleanProperty(), }, }, }, }, + { + name: "additionalProperties bool", + in: &apiextensions.JSONSchemaProps{ + AdditionalProperties: &apiextensions.JSONSchemaPropsOrBool{ + Allows: true, + Schema: nil, + }, + }, + expected: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: nil, + }, + }, + }, + }, { name: "patternProperties", in: &apiextensions.JSONSchemaProps{ @@ -508,15 +522,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { testStr: {Type: "boolean"}, }, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // PatternProperties: map[string]spec.Schema{ - // testStr: *spec.BooleanProperty(), - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "dependencies schema", @@ -527,17 +533,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, }, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // Dependencies: spec.Dependencies{ - // testStr: spec.SchemaOrStringArray{ - // Schema: spec.BooleanProperty(), - // }, - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "dependencies string array", @@ -548,17 +544,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, }, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // Dependencies: spec.Dependencies{ - // testStr: spec.SchemaOrStringArray{ - // Property: []string{testStr2}, - // }, - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "additionalItems", @@ -568,16 +554,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { Schema: &apiextensions.JSONSchemaProps{Type: "boolean"}, }, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // AdditionalItems: &spec.SchemaOrBool{ - // Allows: true, - // Schema: spec.BooleanProperty(), - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "definitions", @@ -586,15 +563,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { testStr: apiextensions.JSONSchemaProps{Type: "boolean"}, }, }, - expected: new(spec.Schema), - // not supported by gnostic - // expected: &spec.Schema{ - // SchemaProps: spec.SchemaProps{ - // Definitions: spec.Definitions{ - // testStr: *spec.BooleanProperty(), - // }, - // }, - // }, + expectError: true, // rejected by kube validation and NewStructural }, { name: "externalDocs", @@ -606,6 +575,7 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, expected: new(spec.Schema). WithExternalDocs(testStr, testStr2), + expectDiff: true, }, { name: "example", @@ -614,115 +584,127 @@ func Test_ConvertJSONSchemaPropsToOpenAPIv2SchemaFuzzing(t *testing.T) { }, expected: new(spec.Schema). WithExample(testStr), + expectDiff: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - out, err := ConvertJSONSchemaPropsToOpenAPIv2Schema(test.in) - if err != nil { - t.Fatalf("unexpected error in converting openapi schema: %v", err) + ss, err := structuralschema.NewStructural(test.in) + if err != nil && !test.expectError { + t.Fatalf("structural schema error: %v", err) + } else if err == nil && test.expectError { + t.Fatalf("expected NewStructural error, but didn't get any") } - if !reflect.DeepEqual(*out, *test.expected) { - t.Errorf("unexpected result:\n want=%v\n got=%v\n\n%s", *test.expected, *out, diff.ObjectDiff(*test.expected, *out)) + + if !test.expectError { + out := ToStructuralOpenAPIV2(ss).ToGoOpenAPI() + if equal := reflect.DeepEqual(*out, *test.expected); !equal && !test.expectDiff { + t.Errorf("unexpected result:\n want=%v\n got=%v\n\n%s", *test.expected, *out, cmp.Diff(*test.expected, *out, cmp.Comparer(refEqual))) + } else if equal && test.expectDiff { + t.Errorf("expected diff, but didn't get any") + } } }) } } +func refEqual(x spec.Ref, y spec.Ref) bool { + return x.String() == y.String() +} + // TestKubeOpenapiRejectionFiltering tests that the CRD openapi schema filtering leads to a spec that the // kube-openapi/pkg/util/proto model code support in version used in Kubernetes 1.13. func TestKubeOpenapiRejectionFiltering(t *testing.T) { for i := 0; i < 10000; i++ { - t.Run(fmt.Sprintf("iteration %d", i), func(t *testing.T) { - f := fuzz.New() - seed := time.Now().UnixNano() - randSource := rand.New(rand.NewSource(seed)) - f.RandSource(randSource) - t.Logf("seed = %d", seed) + f := fuzz.New() + seed := time.Now().UnixNano() + randSource := rand.New(rand.NewSource(seed)) + f.RandSource(randSource) + t.Logf("iteration %d with seed %d", i, seed) - fuzzFuncs(f, func(ref *spec.Ref, c fuzz.Continue, visible bool) { - var url string - if c.RandBool() { - url = fmt.Sprintf("http://%d", c.Intn(100000)) - } else { - url = "#/definitions/test" - } - r, err := spec.NewRef(url) - if err != nil { - t.Fatalf("failed to fuzz ref: %v", err) - } - *ref = r - }) - - // create go-openapi object and fuzz it (we start here because we have the powerful fuzzer already - s := &spec.Schema{} - f.Fuzz(s) - - // convert to apiextensions v1beta1 - bs, err := json.Marshal(s) + fuzzFuncs(f, func(ref *spec.Ref, c fuzz.Continue, visible bool) { + var url string + if c.RandBool() { + url = fmt.Sprintf("http://%d", c.Intn(100000)) + } else { + url = "#/definitions/test" + } + r, err := spec.NewRef(url) if err != nil { - t.Fatal(err) - } - t.Log(string(bs)) - - var schema *apiextensionsv1beta1.JSONSchemaProps - if err := json.Unmarshal(bs, &schema); err != nil { - t.Fatalf("failed to unmarshal JSON into apiextensions/v1beta1: %v", err) - } - - // convert to internal - internalSchema := &apiextensions.JSONSchemaProps{} - if err := apiextensionsv1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(schema, internalSchema, nil); err != nil { - t.Fatalf("failed to convert from apiextensions/v1beta1 to internal: %v", err) - } - - // apply the filter - filtered, err := ConvertJSONSchemaPropsToOpenAPIv2Schema(internalSchema) - if err != nil { - t.Fatalf("failed to filter: %v", err) - } - - // create a doc out of it - filteredSwagger := &spec.Swagger{ - SwaggerProps: spec.SwaggerProps{ - Definitions: spec.Definitions{ - "test": *filtered, - }, - Info: &spec.Info{ - InfoProps: spec.InfoProps{ - Description: "test", - Version: "test", - Title: "test", - }, - }, - Swagger: "2.0", - }, - } - - // convert to JSON - bs, err = json.Marshal(filteredSwagger) - if err != nil { - t.Fatalf("failed to encode filtered to JSON: %v", err) - } - - // unmarshal as yaml - var yml yaml.MapSlice - if err := yaml.Unmarshal(bs, &yml); err != nil { - t.Fatalf("failed to decode filtered JSON by into memory: %v", err) - } - - // create gnostic doc - doc, err := openapi_v2.NewDocument(yml, compiler.NewContext("$root", nil)) - if err != nil { - t.Fatalf("failed to create gnostic doc: %v", err) - } - - // load with kube-openapi/pkg/util/proto - if _, err := proto.NewOpenAPIData(doc); err != nil { - t.Fatalf("failed to convert to kube-openapi/pkg/util/proto model: %v", err) + t.Fatalf("failed to fuzz ref: %v", err) } + *ref = r }) + + // create go-openapi object and fuzz it (we start here because we have the powerful fuzzer already + s := &spec.Schema{} + f.Fuzz(s) + + // convert to apiextensions v1beta1 + bs, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + t.Log(string(bs)) + + var schema *apiextensionsv1beta1.JSONSchemaProps + if err := json.Unmarshal(bs, &schema); err != nil { + t.Fatalf("failed to unmarshal JSON into apiextensions/v1beta1: %v", err) + } + + // convert to internal + internalSchema := &apiextensions.JSONSchemaProps{} + if err := apiextensionsv1beta1.Convert_v1beta1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(schema, internalSchema, nil); err != nil { + t.Fatalf("failed to convert from apiextensions/v1beta1 to internal: %v", err) + } + + // apply the filter + ss, err := structuralschema.NewStructural(internalSchema) + if err != nil { + t.Fatal(err) + } + filtered := ToStructuralOpenAPIV2(ss).ToGoOpenAPI() + + // create a doc out of it + filteredSwagger := &spec.Swagger{ + SwaggerProps: spec.SwaggerProps{ + Definitions: spec.Definitions{ + "test": *filtered, + }, + Info: &spec.Info{ + InfoProps: spec.InfoProps{ + Description: "test", + Version: "test", + Title: "test", + }, + }, + Swagger: "2.0", + }, + } + + // convert to JSON + bs, err = json.Marshal(filteredSwagger) + if err != nil { + t.Fatalf("failed to encode filtered to JSON: %v", err) + } + + // unmarshal as yaml + var yml yaml.MapSlice + if err := yaml.Unmarshal(bs, &yml); err != nil { + t.Fatalf("failed to decode filtered JSON by into memory: %v", err) + } + + // create gnostic doc + doc, err := openapi_v2.NewDocument(yml, compiler.NewContext("$root", nil)) + if err != nil { + t.Fatalf("failed to create gnostic doc: %v", err) + } + + // load with kube-openapi/pkg/util/proto + if _, err := proto.NewOpenAPIData(doc); err != nil { + t.Fatalf("failed to convert to kube-openapi/pkg/util/proto model: %v", err) + } } } @@ -794,9 +776,11 @@ func fuzzFuncs(f *fuzz.Fuzzer, refFunc func(ref *spec.Ref, c fuzz.Continue, visi if p.Default != nil { p.Default = "42" } - if p.Example != nil { - p.Example = "42" - } + p.Example = nil + }, + func(s *spec.SwaggerSchemaProps, c fuzz.Continue) { + // nothing allowed + *s = spec.SwaggerSchemaProps{} }, func(s *spec.SchemaProps, c fuzz.Continue) { // gofuzz is broken and calls this even for *SchemaProps fields, ignoring NilChance, leading to infinite recursion @@ -809,14 +793,26 @@ func fuzzFuncs(f *fuzz.Fuzzer, refFunc func(ref *spec.Ref, c fuzz.Continue, visi c.FuzzNoCustom(s) - // we don't support multi-type schema props yet in apiextensions/v1beta1 - if len(s.Type) > 1 { - s.Type = s.Type[:1] + if c.RandBool() { + types := []string{"object", "array", "boolean", "string", "integer", "number"} + s.Type = []string{types[c.Intn(len(types))]} + } else { + s.Type = nil + } - s := apiextensionsv1beta1.JSONSchemaProps{} - if reflect.TypeOf(s.Type).String() != "string" { - panic(fmt.Errorf("this simplifaction is outdated: apiextensions/v1beta1 types not a single string anymore, but %T", s.Type)) - } + s.ID = "" + s.Ref = spec.Ref{} + s.AdditionalItems = nil + s.Dependencies = nil + s.Schema = "" + s.PatternProperties = nil + s.Definitions = nil + + if len(s.Type) == 1 && s.Type[0] == "array" { + s.Items = &spec.SchemaOrArray{Schema: &spec.Schema{}} + c.Fuzz(s.Items.Schema) + } else { + s.Items = nil } // reset JSON fields to some correct JSON diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go index a83f27e5efb..72ff90993ca 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go @@ -66,5 +66,5 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS CustomResourceValidation: {Default: true, PreRelease: featuregate.Beta}, CustomResourceSubresources: {Default: true, PreRelease: featuregate.Beta}, CustomResourceWebhookConversion: {Default: false, PreRelease: featuregate.Alpha}, - CustomResourcePublishOpenAPI: {Default: false, PreRelease: featuregate.Alpha}, + CustomResourcePublishOpenAPI: {Default: true, PreRelease: featuregate.Beta}, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD index 8830d55815e..252307c8e06 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD @@ -71,7 +71,9 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion:all-srcs", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:all-srcs", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration/storage:all-srcs", ], tags = ["automanaged"], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/BUILD new file mode 100644 index 00000000000..4468e4bff43 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/BUILD @@ -0,0 +1,61 @@ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_test( + name = "go_default_test", + srcs = ["conversion_test.go"], + embed = [":go_default_library"], + tags = ["integration"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration/storage:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = ["webhook.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration/conversion", + importpath = "k8s.io/apiextensions-apiserver/test/integration/conversion", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go new file mode 100644 index 00000000000..192f0b9aada --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go @@ -0,0 +1,898 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" + serveroptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + etcd3watcher "k8s.io/apiserver/pkg/storage/etcd3" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/dynamic" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/utils/pointer" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/test/integration/fixtures" + "k8s.io/apiextensions-apiserver/test/integration/storage" +) + +type Checker func(t *testing.T, ctc *conversionTestContext) + +func checks(checkers ...Checker) []Checker { + return checkers +} + +func TestWebhookConverter(t *testing.T) { + testWebhookConverter(t, false) +} + +func TestWebhookConverterWithPruning(t *testing.T) { + testWebhookConverter(t, true) +} + +func testWebhookConverter(t *testing.T, pruning bool) { + tests := []struct { + group string + handler http.Handler + checks []Checker + }{ + { + group: "noop-converter", + handler: NewObjectConverterWebhookHandler(t, noopConverter), + checks: checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1")), // no v1beta2 as the schema differs + }, + { + group: "nontrivial-converter", + handler: NewObjectConverterWebhookHandler(t, nontrivialConverter), + checks: checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1", "v1beta2"), validateNonTrivialConverted, validateNonTrivialConvertedList, validateStoragePruning), + }, + { + group: "empty-response", + handler: NewReviewWebhookHandler(t, emptyResponseConverter), + checks: checks(expectConversionFailureMessage("empty-response", "expected 1 converted objects")), + }, + { + group: "failure-message", + handler: NewReviewWebhookHandler(t, failureResponseConverter("custom webhook conversion error")), + checks: checks(expectConversionFailureMessage("failure-message", "custom webhook conversion error")), + }, + } + + // TODO: Added for integration testing of conversion webhooks, where decode errors due to conversion webhook failures need to be tested. + // Maybe we should identify conversion webhook related errors in decoding to avoid triggering this? Or maybe having this special casing + // of test cases in production code should be removed? + etcd3watcher.TestOnlySetFatalOnDecodeError(false) + defer etcd3watcher.TestOnlySetFatalOnDecodeError(true) + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() + tearDown, config, options, err := fixtures.StartDefaultServer(t) + if err != nil { + t.Fatal(err) + } + + apiExtensionsClient, err := clientset.NewForConfig(config) + if err != nil { + tearDown() + t.Fatal(err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + tearDown() + t.Fatal(err) + } + defer tearDown() + + crd := multiVersionFixture.DeepCopy() + crd.Spec.PreserveUnknownFields = pointer.BoolPtr(!pruning) + + RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) + restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + if err != nil { + t.Fatal(err) + } + etcdClient, _, err := storage.GetEtcdClients(restOptions.StorageConfig.Transport) + if err != nil { + t.Fatal(err) + } + defer etcdClient.Close() + + etcdObjectReader := storage.NewEtcdObjectReader(etcdClient, &restOptions, crd) + ctcTearDown, ctc := newConversionTestContext(t, apiExtensionsClient, dynamicClient, etcdObjectReader, crd) + defer ctcTearDown() + + // read only object to read at a different version than stored when we need to force conversion + marker, err := ctc.versionedClient("marker", "v1beta1").Create(newConversionMultiVersionFixture("marker", "marker", "v1beta1"), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + t.Run(test.group, func(t *testing.T) { + upCh, handler := closeOnCall(test.handler) + tearDown, webhookClientConfig, err := StartConversionWebhookServer(handler) + if err != nil { + t.Fatal(err) + } + defer tearDown() + + ctc.setConversionWebhook(t, webhookClientConfig) + defer ctc.removeConversionWebhook(t) + + // wait until new webhook is called the first time + if err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) { + _, err := ctc.versionedClient(marker.GetNamespace(), "v1alpha1").Get(marker.GetName(), metav1.GetOptions{}) + select { + case <-upCh: + return true, nil + default: + t.Logf("Waiting for webhook to become effective, getting marker object: %v", err) + return false, nil + } + }); err != nil { + t.Fatal(err) + } + + for i, checkFn := range test.checks { + name := fmt.Sprintf("check-%d", i) + t.Run(name, func(t *testing.T) { + defer ctc.setAndWaitStorageVersion(t, "v1beta1") + ctc.namespace = fmt.Sprintf("webhook-conversion-%s-%s", test.group, name) + checkFn(t, ctc) + }) + } + }) + } +} + +func validateStorageVersion(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + + for _, version := range ctc.crd.Spec.Versions { + t.Run(version.Name, func(t *testing.T) { + name := "storageversion-" + version.Name + client := ctc.versionedClient(ns, version.Name) + obj, err := client.Create(newConversionMultiVersionFixture(ns, name, version.Name), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + ctc.setAndWaitStorageVersion(t, "v1beta2") + + obj, err = client.Get(obj.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + ctc.setAndWaitStorageVersion(t, "v1beta1") + }) + } +} + +// validateMixedStorageVersions ensures that identical custom resources written at different storage versions +// are readable and remain the same. +func validateMixedStorageVersions(versions ...string) func(t *testing.T, ctc *conversionTestContext) { + return func(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + clients := ctc.versionedClients(ns) + + // Create CRs at all storage versions + objNames := []string{} + for _, version := range versions { + ctc.setAndWaitStorageVersion(t, version) + + name := "mixedstorage-stored-as-" + version + obj, err := clients[version].Create(newConversionMultiVersionFixture(ns, name, version), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + objNames = append(objNames, obj.GetName()) + } + + // Ensure copies of an object have the same fields and values at each custom resource definition version regardless of storage version + for clientVersion, client := range clients { + t.Run(clientVersion, func(t *testing.T) { + o1, err := client.Get(objNames[0], metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for _, objName := range objNames[1:] { + o2, err := client.Get(objName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + // ignore metadata for comparison purposes + delete(o1.Object, "metadata") + delete(o2.Object, "metadata") + if !reflect.DeepEqual(o1.Object, o2.Object) { + t.Errorf("Expected custom resource to be same regardless of which storage version is used to create, but got: %s", cmp.Diff(o1, o2)) + } + } + }) + } + } +} + +func validateServed(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + + for _, version := range ctc.crd.Spec.Versions { + t.Run(version.Name, func(t *testing.T) { + name := "served-" + version.Name + client := ctc.versionedClient(ns, version.Name) + obj, err := client.Create(newConversionMultiVersionFixture(ns, name, version.Name), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + ctc.setServed(t, version.Name, false) + ctc.waitForServed(t, version.Name, false, client, obj) + ctc.setServed(t, version.Name, true) + ctc.waitForServed(t, version.Name, true, client, obj) + }) + } +} + +func validateNonTrivialConverted(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + + for _, createVersion := range ctc.crd.Spec.Versions { + t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) { + name := "converted-" + createVersion.Name + client := ctc.versionedClient(ns, createVersion.Name) + + fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name) + if !*ctc.crd.Spec.PreserveUnknownFields { + if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil { + t.Fatal(err) + } + } + if _, err := client.Create(fixture, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + // verify that the right, pruned version is in storage + obj, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name) + if err != nil { + t.Fatal(err) + } + verifyMultiVersionObject(t, "v1beta1", obj) + + for _, getVersion := range ctc.crd.Spec.Versions { + client := ctc.versionedClient(ns, getVersion.Name) + obj, err := client.Get(name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + verifyMultiVersionObject(t, getVersion.Name, obj) + } + }) + } +} + +func validateNonTrivialConvertedList(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + "-list" + + names := sets.String{} + for _, createVersion := range ctc.crd.Spec.Versions { + name := "converted-" + createVersion.Name + client := ctc.versionedClient(ns, createVersion.Name) + fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name) + if !*ctc.crd.Spec.PreserveUnknownFields { + if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil { + t.Fatal(err) + } + } + _, err := client.Create(fixture, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + names.Insert(name) + } + + for _, listVersion := range ctc.crd.Spec.Versions { + t.Run(fmt.Sprintf("listing objects as %s", listVersion.Name), func(t *testing.T) { + client := ctc.versionedClient(ns, listVersion.Name) + obj, err := client.List(metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(obj.Items) != len(ctc.crd.Spec.Versions) { + t.Fatal("unexpected number of items") + } + foundNames := sets.String{} + for _, u := range obj.Items { + foundNames.Insert(u.GetName()) + verifyMultiVersionObject(t, listVersion.Name, &u) + } + if !foundNames.Equal(names) { + t.Errorf("unexpected set of returned items: %s", foundNames.Difference(names)) + } + }) + } +} + +func validateStoragePruning(t *testing.T, ctc *conversionTestContext) { + if *ctc.crd.Spec.PreserveUnknownFields { + return + } + + ns := ctc.namespace + + for _, createVersion := range ctc.crd.Spec.Versions { + t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) { + name := "storagepruning-" + createVersion.Name + client := ctc.versionedClient(ns, createVersion.Name) + + fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name) + if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil { + t.Fatal(err) + } + _, err := client.Create(fixture, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + // verify that the right, pruned version is in storage + obj, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name) + if err != nil { + t.Fatal(err) + } + verifyMultiVersionObject(t, "v1beta1", obj) + + // add garbage and set a label + if err := unstructured.SetNestedField(obj.Object, "foo", "garbage"); err != nil { + t.Fatal(err) + } + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels["mutated"] = "true" + obj.SetLabels(labels) + if err := ctc.etcdObjectReader.SetStoredCustomResource(ns, name, obj); err != nil { + t.Fatal(err) + } + + for _, getVersion := range ctc.crd.Spec.Versions { + client := ctc.versionedClient(ns, getVersion.Name) + obj, err := client.Get(name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + // check that the direct mutation in etcd worked + labels := obj.GetLabels() + if labels["mutated"] != "true" { + t.Errorf("expected object %s in version %s to have label 'mutated=true'", name, getVersion.Name) + } + + verifyMultiVersionObject(t, getVersion.Name, obj) + } + }) + } +} + +func expectConversionFailureMessage(id, message string) func(t *testing.T, ctc *conversionTestContext) { + return func(t *testing.T, ctc *conversionTestContext) { + ns := ctc.namespace + clients := ctc.versionedClients(ns) + var err error + // storage version is v1beta1, so this skips conversion + obj, err := clients["v1beta1"].Create(newConversionMultiVersionFixture(ns, id, "v1beta1"), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + for _, verb := range []string{"get", "list", "create", "udpate", "patch", "delete", "deletecollection"} { + t.Run(verb, func(t *testing.T) { + switch verb { + case "get": + _, err = clients["v1beta2"].Get(obj.GetName(), metav1.GetOptions{}) + case "list": + _, err = clients["v1beta2"].List(metav1.ListOptions{}) + case "create": + _, err = clients["v1beta2"].Create(newConversionMultiVersionFixture(ns, id, "v1beta2"), metav1.CreateOptions{}) + case "update": + _, err = clients["v1beta2"].Update(obj, metav1.UpdateOptions{}) + case "patch": + _, err = clients["v1beta2"].Patch(obj.GetName(), types.MergePatchType, []byte(`{"metadata":{"annotations":{"patch":"true"}}}`), metav1.PatchOptions{}) + case "delete": + err = clients["v1beta2"].Delete(obj.GetName(), &metav1.DeleteOptions{}) + case "deletecollection": + err = clients["v1beta2"].DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{}) + } + + if err == nil { + t.Errorf("expected error with message %s, but got no error", message) + } else if !strings.Contains(err.Error(), message) { + t.Errorf("expected error with message %s, but got %v", message, err) + } + }) + } + for _, subresource := range []string{"status", "scale"} { + for _, verb := range []string{"get", "udpate", "patch"} { + t.Run(fmt.Sprintf("%s-%s", subresource, verb), func(t *testing.T) { + switch verb { + case "create": + _, err = clients["v1beta2"].Create(newConversionMultiVersionFixture(ns, id, "v1beta2"), metav1.CreateOptions{}, subresource) + case "update": + _, err = clients["v1beta2"].Update(obj, metav1.UpdateOptions{}, subresource) + case "patch": + _, err = clients["v1beta2"].Patch(obj.GetName(), types.MergePatchType, []byte(`{"metadata":{"annotations":{"patch":"true"}}}`), metav1.PatchOptions{}, subresource) + } + + if err == nil { + t.Errorf("expected error with message %s, but got no error", message) + } else if !strings.Contains(err.Error(), message) { + t.Errorf("expected error with message %s, but got %v", message, err) + } + }) + } + } + } +} + +func noopConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) { + u := &unstructured.Unstructured{Object: map[string]interface{}{}} + if err := json.Unmarshal(obj.Raw, u); err != nil { + return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err) + } + u.Object["apiVersion"] = desiredAPIVersion + raw, err := json.Marshal(u) + if err != nil { + return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err) + } + return runtime.RawExtension{Raw: raw}, nil +} + +func emptyResponseConverter(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) { + review.Response = &apiextensionsv1beta1.ConversionResponse{ + UID: review.Request.UID, + ConvertedObjects: []runtime.RawExtension{}, + Result: metav1.Status{Status: "Success"}, + } + return review, nil +} + +func failureResponseConverter(message string) func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) { + return func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) { + review.Response = &apiextensionsv1beta1.ConversionResponse{ + UID: review.Request.UID, + ConvertedObjects: []runtime.RawExtension{}, + Result: metav1.Status{Message: message, Status: "Failure"}, + } + return review, nil + } +} + +func nontrivialConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) { + u := &unstructured.Unstructured{Object: map[string]interface{}{}} + if err := json.Unmarshal(obj.Raw, u); err != nil { + return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err) + } + + currentAPIVersion := u.GetAPIVersion() + + if currentAPIVersion == "stable.example.com/v1beta2" && (desiredAPIVersion == "stable.example.com/v1alpha1" || desiredAPIVersion == "stable.example.com/v1beta1") { + u.Object["num"] = u.Object["numv2"] + u.Object["content"] = u.Object["contentv2"] + delete(u.Object, "numv2") + delete(u.Object, "contentv2") + } else if (currentAPIVersion == "stable.example.com/v1alpha1" || currentAPIVersion == "stable.example.com/v1beta1") && desiredAPIVersion == "stable.example.com/v1beta2" { + u.Object["numv2"] = u.Object["num"] + u.Object["contentv2"] = u.Object["content"] + delete(u.Object, "num") + delete(u.Object, "content") + } else if currentAPIVersion == "stable.example.com/v1alpha1" && desiredAPIVersion == "stable.example.com/v1beta1" { + // same schema + } else if currentAPIVersion == "stable.example.com/v1beta1" && desiredAPIVersion == "stable.example.com/v1alpha1" { + // same schema + } else if currentAPIVersion != desiredAPIVersion { + return runtime.RawExtension{}, fmt.Errorf("cannot convert from %s to %s", currentAPIVersion, desiredAPIVersion) + } + u.Object["apiVersion"] = desiredAPIVersion + raw, err := json.Marshal(u) + if err != nil { + return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err) + } + return runtime.RawExtension{Raw: raw}, nil +} + +func newConversionTestContext(t *testing.T, apiExtensionsClient clientset.Interface, dynamicClient dynamic.Interface, etcdObjectReader *storage.EtcdObjectReader, crd *apiextensionsv1beta1.CustomResourceDefinition) (func(), *conversionTestContext) { + crd, err := fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionsClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + tearDown := func() { + if err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionsClient); err != nil { + t.Fatal(err) + } + } + + return tearDown, &conversionTestContext{apiExtensionsClient: apiExtensionsClient, dynamicClient: dynamicClient, crd: crd, etcdObjectReader: etcdObjectReader} +} + +type conversionTestContext struct { + namespace string + apiExtensionsClient clientset.Interface + dynamicClient dynamic.Interface + options *options.CustomResourceDefinitionsServerOptions + crd *apiextensionsv1beta1.CustomResourceDefinition + etcdObjectReader *storage.EtcdObjectReader +} + +func (c *conversionTestContext) versionedClient(ns string, version string) dynamic.ResourceInterface { + gvr := schema.GroupVersionResource{Group: c.crd.Spec.Group, Version: version, Resource: c.crd.Spec.Names.Plural} + if c.crd.Spec.Scope != apiextensionsv1beta1.ClusterScoped { + return c.dynamicClient.Resource(gvr).Namespace(ns) + } + return c.dynamicClient.Resource(gvr) +} + +func (c *conversionTestContext) versionedClients(ns string) map[string]dynamic.ResourceInterface { + ret := map[string]dynamic.ResourceInterface{} + for _, v := range c.crd.Spec.Versions { + ret[v.Name] = c.versionedClient(ns, v.Name) + } + return ret +} + +func (c *conversionTestContext) setConversionWebhook(t *testing.T, webhookClientConfig *apiextensionsv1beta1.WebhookClientConfig) { + crd, err := c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(c.crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{ + Strategy: apiextensionsv1beta1.WebhookConverter, + WebhookClientConfig: webhookClientConfig, + } + crd, err = c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd) + if err != nil { + t.Fatal(err) + } + c.crd = crd + +} + +func (c *conversionTestContext) removeConversionWebhook(t *testing.T) { + crd, err := c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(c.crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{ + Strategy: apiextensionsv1beta1.NoneConverter, + } + + crd, err = c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd) + if err != nil { + t.Fatal(err) + } + c.crd = crd +} + +func (c *conversionTestContext) setAndWaitStorageVersion(t *testing.T, version string) { + c.setStorageVersion(t, version) + + // create probe object. Version should be the default one to avoid webhook calls during test setup. + client := c.versionedClient("probe", "v1beta1") + name := fmt.Sprintf("probe-%v", uuid.NewUUID()) + storageProbe, err := client.Create(newConversionMultiVersionFixture("probe", name, "v1beta1"), metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + // update object continuously and wait for etcd to have the target storage version. + c.waitForStorageVersion(t, version, c.versionedClient(storageProbe.GetNamespace(), "v1beta1"), storageProbe) + + err = client.Delete(name, &metav1.DeleteOptions{}) + if err != nil { + t.Fatal(err) + } +} + +func (c *conversionTestContext) setStorageVersion(t *testing.T, version string) { + crd, err := c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(c.crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for i, v := range crd.Spec.Versions { + crd.Spec.Versions[i].Storage = v.Name == version + } + crd, err = c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd) + if err != nil { + t.Fatal(err) + } + c.crd = crd +} + +func (c *conversionTestContext) waitForStorageVersion(t *testing.T, version string, versionedClient dynamic.ResourceInterface, obj *unstructured.Unstructured) *unstructured.Unstructured { + if err := c.etcdObjectReader.WaitForStorageVersion(version, obj.GetNamespace(), obj.GetName(), 30*time.Second, func() { + if _, err := versionedClient.Patch(obj.GetName(), types.MergePatchType, []byte(`{}`), metav1.PatchOptions{}); err != nil { + t.Fatalf("failed to update object: %v", err) + } + }); err != nil { + t.Fatalf("failed waiting for storage version %s: %v", version, err) + } + + t.Logf("Effective storage version: %s", version) + + return obj +} + +func (c *conversionTestContext) setServed(t *testing.T, version string, served bool) { + crd, err := c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(c.crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for i, v := range crd.Spec.Versions { + if v.Name == version { + crd.Spec.Versions[i].Served = served + } + } + crd, err = c.apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd) + if err != nil { + t.Fatal(err) + } + c.crd = crd +} + +func (c *conversionTestContext) waitForServed(t *testing.T, version string, served bool, versionedClient dynamic.ResourceInterface, obj *unstructured.Unstructured) { + timeout := 30 * time.Second + waitCh := time.After(timeout) + for { + obj, err := versionedClient.Get(obj.GetName(), metav1.GetOptions{}) + if (err == nil && served) || (errors.IsNotFound(err) && served == false) { + return + } + select { + case <-waitCh: + t.Fatalf("Timed out after %v waiting for CRD served=%t for version %s for %v. Last error: %v", timeout, served, version, obj, err) + case <-time.After(10 * time.Millisecond): + } + } +} + +var multiVersionFixture = &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "multiversion.stable.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "stable.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "multiversion", + Singular: "multiversion", + Kind: "MultiVersion", + ShortNames: []string{"mv"}, + ListKind: "MultiVersionList", + Categories: []string{"all"}, + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + // storage version, same schema as v1alpha1 + Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "content": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "key": {Type: "string"}, + }, + }, + "num": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "num1": {Type: "integer"}, + "num2": {Type: "integer"}, + }, + }, + }, + }, + }, + }, + { + // same schema as v1beta1 + Name: "v1alpha1", + Served: true, + Storage: false, + Schema: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "content": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "key": {Type: "string"}, + }, + }, + "num": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "num1": {Type: "integer"}, + "num2": {Type: "integer"}, + }, + }, + }, + }, + }, + }, + { + // different schema than v1beta1 and v1alpha1 + Name: "v1beta2", + Served: true, + Storage: false, + Schema: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "contentv2": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "key": {Type: "string"}, + }, + }, + "numv2": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "num1": {Type: "integer"}, + "num2": {Type: "integer"}, + }, + }, + }, + }, + }, + }, + }, + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.num.num1", + StatusReplicasPath: ".status.num.num2", + }, + }, + }, +} + +func newConversionMultiVersionFixture(namespace, name, version string) *unstructured.Unstructured { + u := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "stable.example.com/" + version, + "kind": "MultiVersion", + "metadata": map[string]interface{}{ + "namespace": namespace, + "name": name, + }, + }, + } + + switch version { + case "v1alpha1": + u.Object["content"] = map[string]interface{}{ + "key": "value", + } + u.Object["num"] = map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + } + case "v1beta1": + u.Object["content"] = map[string]interface{}{ + "key": "value", + } + u.Object["num"] = map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + } + case "v1beta2": + u.Object["contentv2"] = map[string]interface{}{ + "key": "value", + } + u.Object["numv2"] = map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + } + default: + panic(fmt.Sprintf("unknown version %s", version)) + } + + return u +} + +func verifyMultiVersionObject(t *testing.T, v string, obj *unstructured.Unstructured) { + j := runtime.DeepCopyJSON(obj.Object) + + if expected := "stable.example.com/" + v; obj.GetAPIVersion() != expected { + t.Errorf("unexpected apiVersion %q, expected %q", obj.GetAPIVersion(), expected) + return + } + + delete(j, "metadata") + + var expected = map[string]map[string]interface{}{ + "v1alpha1": { + "apiVersion": "stable.example.com/v1alpha1", + "kind": "MultiVersion", + "content": map[string]interface{}{ + "key": "value", + }, + "num": map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + }, + }, + "v1beta1": { + "apiVersion": "stable.example.com/v1beta1", + "kind": "MultiVersion", + "content": map[string]interface{}{ + "key": "value", + }, + "num": map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + }, + }, + "v1beta2": { + "apiVersion": "stable.example.com/v1beta2", + "kind": "MultiVersion", + "contentv2": map[string]interface{}{ + "key": "value", + }, + "numv2": map[string]interface{}{ + "num1": int64(1), + "num2": int64(1000000), + }, + }, + } + if !reflect.DeepEqual(expected[v], j) { + t.Errorf("unexpected %s object: %s", v, cmp.Diff(expected[v], j)) + } +} + +func closeOnCall(h http.Handler) (chan struct{}, http.Handler) { + ch := make(chan struct{}) + once := sync.Once{} + return ch, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + once.Do(func() { + close(ch) + }) + h.ServeHTTP(w, r) + }) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/webhook.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/webhook.go new file mode 100644 index 00000000000..e1dcfb84a33 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/webhook.go @@ -0,0 +1,166 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// StartConversionWebhookServer starts an http server with the provided handler and returns the WebhookClientConfig +// needed to configure a CRD to use this conversion webhook as its converter. +func StartConversionWebhookServer(handler http.Handler) (func(), *apiextensionsv1beta1.WebhookClientConfig, error) { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(localhostCert) { + return nil, nil, fmt.Errorf("failed to append Cert from PEM") + } + cert, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to build cert with error: %+v", err) + } + + webhookMux := http.NewServeMux() + webhookMux.Handle("/convert", handler) + webhookServer := httptest.NewUnstartedServer(webhookMux) + webhookServer.TLS = &tls.Config{ + RootCAs: roots, + Certificates: []tls.Certificate{cert}, + } + webhookServer.StartTLS() + endpoint := webhookServer.URL + "/convert" + webhookConfig := &apiextensionsv1beta1.WebhookClientConfig{ + CABundle: localhostCert, + URL: &endpoint, + } + + // StartTLS returns immediately, there is a small chance of a race to avoid. + if err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) { + _, err := webhookServer.Client().Get(webhookServer.URL) // even a 404 is fine + return err == nil, nil + }); err != nil { + webhookServer.Close() + return nil, nil, err + } + + return webhookServer.Close, webhookConfig, nil +} + +// ReviewConverterFunc converts an entire ConversionReview. +type ReviewConverterFunc func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) + +// NewReviewWebhookHandler creates a handler that delegates the review conversion to the provided ReviewConverterFunc. +func NewReviewWebhookHandler(t *testing.T, converterFunc ReviewConverterFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + data, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { + t.Errorf("contentType=%s, expect application/json", contentType) + return + } + + review := apiextensionsv1beta1.ConversionReview{} + if err := json.Unmarshal(data, &review); err != nil { + t.Errorf("Fail to deserialize object: %s with error: %v", string(data), err) + http.Error(w, err.Error(), 400) + return + } + + review, err = converterFunc(review) + if err != nil { + t.Errorf("Error converting review: %v", err) + http.Error(w, err.Error(), 500) + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(review); err != nil { + t.Errorf("Marshal of response failed with error: %v", err) + } + }) +} + +// ObjectConverterFunc converts a single custom resource to the desiredAPIVersion and returns it or returns an error. +type ObjectConverterFunc func(desiredAPIVersion string, customResource runtime.RawExtension) (runtime.RawExtension, error) + +// NewObjectConverterWebhookHandler creates a handler that delegates custom resource conversion to the provided ConverterFunc. +func NewObjectConverterWebhookHandler(t *testing.T, converterFunc ObjectConverterFunc) http.Handler { + return NewReviewWebhookHandler(t, func(review apiextensionsv1beta1.ConversionReview) (apiextensionsv1beta1.ConversionReview, error) { + converted := []runtime.RawExtension{} + errMsgs := []string{} + for _, obj := range review.Request.Objects { + convertedObj, err := converterFunc(review.Request.DesiredAPIVersion, obj) + if err != nil { + errMsgs = append(errMsgs, err.Error()) + } + + converted = append(converted, convertedObj) + } + + review.Response = &apiextensionsv1beta1.ConversionResponse{ + UID: review.Request.UID, + ConvertedObjects: converted, + } + if len(errMsgs) == 0 { + review.Response.Result = metav1.Status{Status: "Success"} + } else { + review.Response.Result = metav1.Status{Status: "Failure", Message: strings.Join(errMsgs, ", ")} + } + return review, nil + }) +} + +// localhostCert was generated from crypto/tls/generate_cert.go with the following command: +// go run generate_cert.go --rsa-bits 512 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +var localhostCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBjzCCATmgAwIBAgIRAKpi2WmTcFrVjxrl5n5YDUEwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2 +MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzBcMA0GCSqGSIb3DQEBAQUAA0sAMEgC +QQC9fEbRszP3t14Gr4oahV7zFObBI4TfA5i7YnlMXeLinb7MnvT4bkfOJzE6zktn +59zP7UiHs3l4YOuqrjiwM413AgMBAAGjaDBmMA4GA1UdDwEB/wQEAwICpDATBgNV +HSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MC4GA1UdEQQnMCWCC2V4 +YW1wbGUuY29thwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEBCwUA +A0EAUsVE6KMnza/ZbodLlyeMzdo7EM/5nb5ywyOxgIOCf0OOLHsPS9ueGLQX9HEG +//yjTXuhNcUugExIjM/AIwAZPQ== +-----END CERTIFICATE-----`) + +// localhostKey is the private key for localhostCert. +var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIBOwIBAAJBAL18RtGzM/e3XgavihqFXvMU5sEjhN8DmLtieUxd4uKdvsye9Phu +R84nMTrOS2fn3M/tSIezeXhg66quOLAzjXcCAwEAAQJBAKcRxH9wuglYLBdI/0OT +BLzfWPZCEw1vZmMR2FF1Fm8nkNOVDPleeVGTWoOEcYYlQbpTmkGSxJ6ya+hqRi6x +goECIQDx3+X49fwpL6B5qpJIJMyZBSCuMhH4B7JevhGGFENi3wIhAMiNJN5Q3UkL +IuSvv03kaPR5XVQ99/UeEetUgGvBcABpAiBJSBzVITIVCGkGc7d+RCf49KTCIklv +bGWObufAR8Ni4QIgWpILjW8dkGg8GOUZ0zaNA6Nvt6TIv2UWGJ4v5PoV98kCIQDx +rIiZs5QbKdycsv9gQJzwQAogC8o04X3Zz3dsoX+h4A== +-----END RSA PRIVATE KEY-----`) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go index 0a3d6444e9d..d2f487051fc 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures/server.go @@ -31,7 +31,7 @@ import ( ) // StartDefaultServer starts a test server. -func StartDefaultServer(t servertesting.Logger) (func(), *rest.Config, *options.CustomResourceDefinitionsServerOptions, error) { +func StartDefaultServer(t servertesting.Logger, flags ...string) (func(), *rest.Config, *options.CustomResourceDefinitionsServerOptions, error) { // create kubeconfig which will not actually be used. But authz/authn needs it to startup. fakeKubeConfig, err := ioutil.TempFile("", "kubeconfig") fakeKubeConfig.WriteString(` @@ -55,15 +55,16 @@ users: `) fakeKubeConfig.Close() - s, err := servertesting.StartTestServer(t, nil, []string{ + s, err := servertesting.StartTestServer(t, nil, append([]string{ "--etcd-prefix", uuid.New(), "--etcd-servers", strings.Join(IntegrationEtcdServers(), ","), "--authentication-skip-lookup", "--authentication-kubeconfig", fakeKubeConfig.Name(), "--authorization-kubeconfig", fakeKubeConfig.Name(), "--kubeconfig", fakeKubeConfig.Name(), - "--disable-admission-plugins", "NamespaceLifecycle,MutatingAdmissionWebhook,ValidatingAdmissionWebhook", - }, nil) + "--disable-admission-plugins", "NamespaceLifecycle,MutatingAdmissionWebhook,ValidatingAdmissionWebhook"}, + flags..., + ), nil) if err != nil { os.Remove(fakeKubeConfig.Name()) return nil, nil, nil, err @@ -78,8 +79,8 @@ users: } // StartDefaultServerWithClients starts a test server and returns clients for it. -func StartDefaultServerWithClients(t servertesting.Logger) (func(), clientset.Interface, dynamic.Interface, error) { - tearDown, config, _, err := StartDefaultServer(t) +func StartDefaultServerWithClients(t servertesting.Logger, extraFlags ...string) (func(), clientset.Interface, dynamic.Interface, error) { + tearDown, config, _, err := StartDefaultServer(t, extraFlags...) if err != nil { return nil, nil, nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD new file mode 100644 index 00000000000..e3fd12256e2 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["objectreader.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/test/integration/storage", + importpath = "k8s.io/apiextensions-apiserver/test/integration/storage", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", + "//vendor/github.com/coreos/etcd/clientv3:go_default_library", + "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go new file mode 100644 index 00000000000..d784458ce94 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/storage/objectreader.go @@ -0,0 +1,125 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "context" + "encoding/json" + "fmt" + "path" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/pkg/transport" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apiserver/pkg/registry/generic" + "k8s.io/apiserver/pkg/storage/storagebackend" +) + +// EtcdObjectReader provides direct access to custom resource objects stored in etcd. +type EtcdObjectReader struct { + etcdClient *clientv3.Client + storagePrefix string + crd *apiextensionsv1beta1.CustomResourceDefinition +} + +// NewEtcdObjectReader creates a reader for accessing custom resource objects directly from etcd. +func NewEtcdObjectReader(etcdClient *clientv3.Client, restOptions *generic.RESTOptions, crd *apiextensionsv1beta1.CustomResourceDefinition) *EtcdObjectReader { + return &EtcdObjectReader{etcdClient, restOptions.StorageConfig.Prefix, crd} +} + +// WaitForStorageVersion calls the updateObjFn periodically and waits for the version of the custom resource stored in etcd to be set to the provided version. +// Typically updateObjFn should perform a noop update to the object so that when stored version of a CRD changes, the object is written at the updated storage version. +// If the timeout is exceeded a error is returned. +// This is useful when updating the stored version of an existing CRD because the update does not take effect immediately. +func (s *EtcdObjectReader) WaitForStorageVersion(version string, ns, name string, timeout time.Duration, updateObjFn func()) error { + waitCh := time.After(timeout) + for { + storage, err := s.GetStoredCustomResource(ns, name) + if err != nil { + return err + } + if storage.GetObjectKind().GroupVersionKind().Version == version { + return nil + } + select { + case <-waitCh: + return fmt.Errorf("timed out after %v waiting for storage version to be %s for object (namespace:%s name:%s)", timeout, version, ns, name) + case <-time.After(10 * time.Millisecond): + updateObjFn() + } + } +} + +// GetStoredCustomResource gets the storage representation of a custom resource from etcd. +func (s *EtcdObjectReader) GetStoredCustomResource(ns, name string) (*unstructured.Unstructured, error) { + key := path.Join("/", s.storagePrefix, s.crd.Spec.Group, s.crd.Spec.Names.Plural, ns, name) + resp, err := s.etcdClient.KV.Get(context.Background(), key) + if err != nil { + return nil, fmt.Errorf("error getting storage object %s, %s from etcd at key %s: %v", ns, name, key, err) + } + if len(resp.Kvs) == 0 { + return nil, fmt.Errorf("no storage object found for %s, %s in etcd for key %s", ns, name, key) + } + raw := resp.Kvs[0].Value + u := &unstructured.Unstructured{Object: map[string]interface{}{}} + if err := json.Unmarshal(raw, u); err != nil { + return nil, fmt.Errorf("error deserializing object %s: %v", string(raw), err) + } + return u, nil +} + +// SetStoredCustomResource writes the storage representation of a custom resource to etcd. +func (s *EtcdObjectReader) SetStoredCustomResource(ns, name string, obj *unstructured.Unstructured) error { + bs, err := obj.MarshalJSON() + if err != nil { + return err + } + + key := path.Join("/", s.storagePrefix, s.crd.Spec.Group, s.crd.Spec.Names.Plural, ns, name) + if _, err := s.etcdClient.KV.Put(context.Background(), key, string(bs)); err != nil { + return fmt.Errorf("error setting storage object %s, %s from etcd at key %s: %v", ns, name, key, err) + } + return nil +} + +// GetEtcdClients returns an initialized clientv3.Client and clientv3.KV. +func GetEtcdClients(config storagebackend.TransportConfig) (*clientv3.Client, clientv3.KV, error) { + tlsInfo := transport.TLSInfo{ + CertFile: config.CertFile, + KeyFile: config.KeyFile, + CAFile: config.CAFile, + } + + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, nil, err + } + + cfg := clientv3.Config{ + Endpoints: config.ServerList, + TLS: tlsConfig, + } + + c, err := clientv3.New(cfg) + if err != nil { + return nil, nil, err + } + + return c, clientv3.NewKV(c), nil +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go index d770f425911..6f3cf72b49b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/subresources_test.go @@ -534,7 +534,7 @@ func TestValidateOnlyStatus(t *testing.T) { } createdNoxuInstance, err = noxuResourceClient.UpdateStatus(createdNoxuInstance, metav1.UpdateOptions{}) if err != nil { - t.Errorf("unexpected error: %v", err) + t.Fatalf("unexpected error: %v", err) } // update with .status.num = 15, expecting an error diff --git a/staging/src/k8s.io/apimachinery/go.mod b/staging/src/k8s.io/apimachinery/go.mod index b47358be679..e6cdb7b438c 100644 --- a/staging/src/k8s.io/apimachinery/go.mod +++ b/staging/src/k8s.io/apimachinery/go.mod @@ -31,7 +31,7 @@ require ( golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db // indirect gopkg.in/inf.v0 v0.9.0 gopkg.in/yaml.v2 v2.2.1 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 sigs.k8s.io/yaml v1.1.0 ) diff --git a/staging/src/k8s.io/apimachinery/go.sum b/staging/src/k8s.io/apimachinery/go.sum index 7a547e60e59..1336ae6cea2 100644 --- a/staging/src/k8s.io/apimachinery/go.sum +++ b/staging/src/k8s.io/apimachinery/go.sum @@ -65,8 +65,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index 5ca2fb7d7f4..ab43946a10a 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -61,7 +61,7 @@ require ( k8s.io/apimachinery v0.0.0 k8s.io/client-go v0.0.0 k8s.io/component-base v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 k8s.io/utils v0.0.0-20190221042446-c2654d5206da sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2 diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index e2913a4526c..ba15f8ecc1f 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -192,8 +192,8 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go index 4f8d07a42dc..9419af9ed37 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go @@ -29,6 +29,8 @@ import ( ) // DeprecatedInsecureServingInfo is the main context object for the insecure http server. +// HTTP does NOT include authentication or authorization. +// You shouldn't be using this. It makes sig-auth sad. type DeprecatedInsecureServingInfo struct { // Listener is the secure server network listener. Listener net.Listener diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go index 15437f27e74..f2b16f3bd75 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go @@ -56,9 +56,15 @@ func testingDeferOnDecodeError() { func init() { // check to see if we are running in a test environment + TestOnlySetFatalOnDecodeError(true) fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR")) } +// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks. +func TestOnlySetFatalOnDecodeError(b bool) { + fatalOnDecodeError = b +} + type watcher struct { client *clientv3.Client codec runtime.Codec diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD index decb48beb59..9c4716b41ff 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD @@ -8,8 +8,17 @@ load( go_test( name = "go_default_test", - srcs = ["transformer_test.go"], + srcs = [ + "metrics_test.go", + "transformer_test.go", + ], embed = [":go_default_library"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library", + "//vendor/google.golang.org/grpc/codes:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + ], ) go_library( @@ -20,7 +29,10 @@ go_library( ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/storage/value", importpath = "k8s.io/apiserver/pkg/storage/value", - deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"], + deps = [ + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/google.golang.org/grpc/status:go_default_library", + ], ) filegroup( diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go index c9dc66a8f35..f827bffaf92 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go @@ -1,5 +1,5 @@ /* -Copyright 2017 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( "sync" "time" + "google.golang.org/grpc/status" + "github.com/prometheus/client_golang/prometheus" ) @@ -53,12 +55,23 @@ var ( }, []string{"transformation_type"}, ) - transformerFailuresTotal = prometheus.NewCounterVec( + + transformerOperationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "transformation_operations_total", + Help: "Total number of transformations.", + }, + []string{"transformation_type", "status"}, + ) + + deprecatedTransformerFailuresTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: "transformation_failures_total", - Help: "Total number of failed transformation operations.", + Help: "(Deprecated) Total number of failed transformation operations.", }, []string{"transformation_type"}, ) @@ -106,7 +119,8 @@ func RegisterMetrics() { registerMetrics.Do(func() { prometheus.MustRegister(transformerLatencies) prometheus.MustRegister(deprecatedTransformerLatencies) - prometheus.MustRegister(transformerFailuresTotal) + prometheus.MustRegister(transformerOperationsTotal) + prometheus.MustRegister(deprecatedTransformerFailuresTotal) prometheus.MustRegister(envelopeTransformationCacheMissTotal) prometheus.MustRegister(dataKeyGenerationLatencies) prometheus.MustRegister(deprecatedDataKeyGenerationLatencies) @@ -115,14 +129,17 @@ func RegisterMetrics() { } // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations. +// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead. func RecordTransformation(transformationType string, start time.Time, err error) { - if err != nil { - transformerFailuresTotal.WithLabelValues(transformationType).Inc() - return - } + transformerOperationsTotal.WithLabelValues(transformationType, status.Code(err).String()).Inc() - transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start)) - deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) + switch { + case err == nil: + transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start)) + deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) + default: + deprecatedTransformerFailuresTotal.WithLabelValues(transformationType).Inc() + } } // RecordCacheMiss records a miss on Key Encryption Key(KEK) - call to KMS was required to decrypt KEK. diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go new file mode 100644 index 00000000000..aba5454591b --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "errors" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestTotals(t *testing.T) { + testCases := []struct { + desc string + metrics []string + error error + want string + }{ + { + desc: "non-status error", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + error: errors.New("foo"), + want: ` + # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # TYPE apiserver_storage_transformation_failures_total counter + apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt"} 1 +`, + }, + { + desc: "error is nil", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + want: ` + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt"} 1 +`, + }, + { + desc: "status error from kms-plugin", + metrics: []string{ + "apiserver_storage_transformation_operations_total", + "apiserver_storage_transformation_failures_total", + }, + error: status.Error(codes.FailedPrecondition, "foo"), + want: ` + # HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. + # TYPE apiserver_storage_transformation_failures_total counter + apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 + # HELP apiserver_storage_transformation_operations_total Total number of transformations. + # TYPE apiserver_storage_transformation_operations_total counter + apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt"} 1 +`, + }, + } + + RegisterMetrics() + + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + RecordTransformation("encrypt", time.Now(), tt.error) + defer transformerOperationsTotal.Reset() + defer deprecatedTransformerFailuresTotal.Reset() + if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/staging/src/k8s.io/cli-runtime/go.sum b/staging/src/k8s.io/cli-runtime/go.sum index 82c5e91fad2..10482de0a13 100644 --- a/staging/src/k8s.io/cli-runtime/go.sum +++ b/staging/src/k8s.io/cli-runtime/go.sum @@ -110,8 +110,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/client-go/go.mod b/staging/src/k8s.io/client-go/go.mod index 79d1390475a..cfd038a1784 100644 --- a/staging/src/k8s.io/client-go/go.mod +++ b/staging/src/k8s.io/client-go/go.mod @@ -28,7 +28,7 @@ require ( google.golang.org/appengine v1.5.0 // indirect k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/utils v0.0.0-20190221042446-c2654d5206da sigs.k8s.io/yaml v1.1.0 ) diff --git a/staging/src/k8s.io/client-go/go.sum b/staging/src/k8s.io/client-go/go.sum index 541a1e6b904..5e2bb548d9c 100644 --- a/staging/src/k8s.io/client-go/go.sum +++ b/staging/src/k8s.io/client-go/go.sum @@ -91,8 +91,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/client-go/tools/auth/clientauth.go b/staging/src/k8s.io/client-go/tools/auth/clientauth.go index 20339ab9d81..c341726774f 100644 --- a/staging/src/k8s.io/client-go/tools/auth/clientauth.go +++ b/staging/src/k8s.io/client-go/tools/auth/clientauth.go @@ -105,7 +105,7 @@ func LoadFromFile(path string) (*Info, error) { // The fields of client.Config with a corresponding field in the Info are set // with the value from the Info. func (info Info) MergeWithConfig(c restclient.Config) (restclient.Config, error) { - var config restclient.Config = c + var config = c config.Username = info.User config.Password = info.Password config.CAFile = info.CAFile @@ -118,6 +118,7 @@ func (info Info) MergeWithConfig(c restclient.Config) (restclient.Config, error) return config, nil } +// Complete returns true if the Kubernetes API authorization info is complete. func (info Info) Complete() bool { return len(info.User) > 0 || len(info.CertFile) > 0 || diff --git a/staging/src/k8s.io/client-go/tools/cache/reflector.go b/staging/src/k8s.io/client-go/tools/cache/reflector.go index 72a69ea6e98..2daa44ba54f 100644 --- a/staging/src/k8s.io/client-go/tools/cache/reflector.go +++ b/staging/src/k8s.io/client-go/tools/cache/reflector.go @@ -83,7 +83,7 @@ var ( // NewNamespaceKeyedIndexerAndReflector creates an Indexer and a Reflector // The indexer is configured to key on namespace func NewNamespaceKeyedIndexerAndReflector(lw ListerWatcher, expectedType interface{}, resyncPeriod time.Duration) (indexer Indexer, reflector *Reflector) { - indexer = NewIndexer(MetaNamespaceKeyFunc, Indexers{"namespace": MetaNamespaceIndexFunc}) + indexer = NewIndexer(MetaNamespaceKeyFunc, Indexers{NamespaceIndex: MetaNamespaceIndexFunc}) reflector = NewReflector(lw, expectedType, indexer, resyncPeriod) return indexer, reflector } diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index 4b0948c3b51..53523ddddc3 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -19,12 +19,13 @@ limitations under the License. // election state. This implementation does not guarantee that only one // client is acting as a leader (a.k.a. fencing). // -// A client observes timestamps captured locally to infer the state of the -// leader election. Thus the implementation is tolerant to arbitrary clock -// skew, but is not tolerant to arbitrary clock skew rate. Timestamp(renew time) -// is not meaningful if it was collected on another machine. The implementation -// of this client only acts on locally collected timestamps and cannot rely on -// the accuracy of timestamp in the record for correctness. +// A client only acts on timestamps captured locally to infer the state of the +// leader election. The client does not consider timestamps in the leader +// election record to be accurate because these timestamps may not have been +// produced by a local clock. The implemention does not depend on their +// accuracy and only uses their change to indicate that another client has +// renewed the leader lease. Thus the implementation is tolerant to arbitrary +// clock skew, but is not tolerant to arbitrary clock skew rate. // // However the level of tolerance to skew rate can be configured by setting // RenewDeadline and LeaseDuration appropriately. The tolerance expressed as a @@ -107,21 +108,27 @@ type LeaderElectionConfig struct { // LeaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. This is measured against time of - // last observed ack. A client needs to wait a full LeaseDuration without - // observing a change to the record before it can attempt to take over even - // when a client with a different identity against the record's starts and - // the renew time in the record is older than LeaseDuration. A.k.a., when - // all clients are shutdown and after at least a LeaseDuration, clients - // started with different identities against the record's must wait a full - // LeaseDuration before acquiring a lock. Thus LeaseDuration should be as - // short as possible to avoid a possible long waiting. LeaseDuration is 15 - // seconds in core Kubernetes components. + // last observed ack. + // + // A client needs to wait a full LeaseDuration without observing a change to + // the record before it can attempt to take over. When all clients are + // shutdown and a new set of clients are started with different names against + // the same leader record, they must wait the full LeaseDuration before + // attempting to acquire the lease. Thus LeaseDuration should be as short as + // possible (within your tolerance for clock skew rate) to avoid a possible + // long waits in the scenario. + // + // Core clients default this value to 15 seconds. LeaseDuration time.Duration // RenewDeadline is the duration that the acting master will retry // refreshing leadership before giving up. + // + // Core clients default this value to 10 seconds. RenewDeadline time.Duration // RetryPeriod is the duration the LeaderElector clients should wait // between tries of actions. + // + // Core clients default this value to 2 seconds. RetryPeriod time.Duration // Callbacks are callbacks that are triggered during certain lifecycle diff --git a/staging/src/k8s.io/client-go/tools/portforward/portforward.go b/staging/src/k8s.io/client-go/tools/portforward/portforward.go index a50a9973ee2..4ab72bb4f3c 100644 --- a/staging/src/k8s.io/client-go/tools/portforward/portforward.go +++ b/staging/src/k8s.io/client-go/tools/portforward/portforward.go @@ -33,8 +33,8 @@ import ( "k8s.io/apimachinery/pkg/util/runtime" ) +// PortForwardProtocolV1Name is the subprotocol used for port forwarding. // TODO move to API machinery and re-unify with kubelet/server/portfoward -// The subprotocol "portforward.k8s.io" is used for port forwarding. const PortForwardProtocolV1Name = "portforward.k8s.io" // PortForwarder knows how to listen for local connections and forward them to @@ -401,6 +401,7 @@ func (pf *PortForwarder) handleConnection(conn net.Conn, port ForwardedPort) { } } +// Close stops all listeners of PortForwarder. func (pf *PortForwarder) Close() { // stop all listeners for _, l := range pf.listeners { diff --git a/staging/src/k8s.io/client-go/tools/watch/informerwatcher.go b/staging/src/k8s.io/client-go/tools/watch/informerwatcher.go index 4ccc4b49a9c..4e0a400bb55 100644 --- a/staging/src/k8s.io/client-go/tools/watch/informerwatcher.go +++ b/staging/src/k8s.io/client-go/tools/watch/informerwatcher.go @@ -18,42 +18,86 @@ package watch import ( "sync" - "sync/atomic" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" ) -func newTicketer() *ticketer { - return &ticketer{ +func newEventProcessor(out chan<- watch.Event) *eventProcessor { + return &eventProcessor{ + out: out, cond: sync.NewCond(&sync.Mutex{}), + done: make(chan struct{}), } } -type ticketer struct { - counter uint64 +// eventProcessor buffers events and writes them to an out chan when a reader +// is waiting. Because of the requirement to buffer events, it synchronizes +// input with a condition, and synchronizes output with a channels. It needs to +// be able to yield while both waiting on an input condition and while blocked +// on writing to the output channel. +type eventProcessor struct { + out chan<- watch.Event - cond *sync.Cond - current uint64 + cond *sync.Cond + buff []watch.Event + + done chan struct{} } -func (t *ticketer) GetTicket() uint64 { - // -1 to start from 0 - return atomic.AddUint64(&t.counter, 1) - 1 +func (e *eventProcessor) run() { + for { + batch := e.takeBatch() + e.writeBatch(batch) + if e.stopped() { + return + } + } } -func (t *ticketer) WaitForTicket(ticket uint64, f func()) { - t.cond.L.Lock() - defer t.cond.L.Unlock() - for ticket != t.current { - t.cond.Wait() +func (e *eventProcessor) takeBatch() []watch.Event { + e.cond.L.Lock() + defer e.cond.L.Unlock() + + for len(e.buff) == 0 && !e.stopped() { + e.cond.Wait() } - f() + batch := e.buff + e.buff = nil + return batch +} - t.current++ - t.cond.Broadcast() +func (e *eventProcessor) writeBatch(events []watch.Event) { + for _, event := range events { + select { + case e.out <- event: + case <-e.done: + return + } + } +} + +func (e *eventProcessor) push(event watch.Event) { + e.cond.L.Lock() + defer e.cond.L.Unlock() + defer e.cond.Signal() + e.buff = append(e.buff, event) +} + +func (e *eventProcessor) stopped() bool { + select { + case <-e.done: + return true + default: + return false + } +} + +func (e *eventProcessor) stop() { + close(e.done) + e.cond.Signal() } // NewIndexerInformerWatcher will create an IndexerInformer and wrap it into watch.Interface @@ -61,55 +105,44 @@ func (t *ticketer) WaitForTicket(ticket uint64, f func()) { // it also returns a channel you can use to wait for the informers to fully shutdown. func NewIndexerInformerWatcher(lw cache.ListerWatcher, objType runtime.Object) (cache.Indexer, cache.Controller, watch.Interface, <-chan struct{}) { ch := make(chan watch.Event) - doneCh := make(chan struct{}) w := watch.NewProxyWatcher(ch) - t := newTicketer() + e := newEventProcessor(ch) indexer, informer := cache.NewIndexerInformer(lw, objType, 0, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - go t.WaitForTicket(t.GetTicket(), func() { - select { - case ch <- watch.Event{ - Type: watch.Added, - Object: obj.(runtime.Object), - }: - case <-w.StopChan(): - } + e.push(watch.Event{ + Type: watch.Added, + Object: obj.(runtime.Object), }) }, UpdateFunc: func(old, new interface{}) { - go t.WaitForTicket(t.GetTicket(), func() { - select { - case ch <- watch.Event{ - Type: watch.Modified, - Object: new.(runtime.Object), - }: - case <-w.StopChan(): - } + e.push(watch.Event{ + Type: watch.Modified, + Object: new.(runtime.Object), }) }, DeleteFunc: func(obj interface{}) { - go t.WaitForTicket(t.GetTicket(), func() { - staleObj, stale := obj.(cache.DeletedFinalStateUnknown) - if stale { - // We have no means of passing the additional information down using watch API based on watch.Event - // but the caller can filter such objects by checking if metadata.deletionTimestamp is set - obj = staleObj - } + staleObj, stale := obj.(cache.DeletedFinalStateUnknown) + if stale { + // We have no means of passing the additional information down using + // watch API based on watch.Event but the caller can filter such + // objects by checking if metadata.deletionTimestamp is set + obj = staleObj + } - select { - case ch <- watch.Event{ - Type: watch.Deleted, - Object: obj.(runtime.Object), - }: - case <-w.StopChan(): - } + e.push(watch.Event{ + Type: watch.Deleted, + Object: obj.(runtime.Object), }) }, }, cache.Indexers{}) + go e.run() + + doneCh := make(chan struct{}) go func() { defer close(doneCh) + defer e.stop() informer.Run(w.StopChan()) }() diff --git a/staging/src/k8s.io/client-go/tools/watch/informerwatcher_test.go b/staging/src/k8s.io/client-go/tools/watch/informerwatcher_test.go index 051898654f1..d6e1e8e223f 100644 --- a/staging/src/k8s.io/client-go/tools/watch/informerwatcher_test.go +++ b/staging/src/k8s.io/client-go/tools/watch/informerwatcher_test.go @@ -17,8 +17,9 @@ limitations under the License. package watch import ( - "math/rand" + "context" "reflect" + goruntime "runtime" "sort" "testing" "time" @@ -28,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/watch" fakeclientset "k8s.io/client-go/kubernetes/fake" @@ -35,6 +37,86 @@ import ( "k8s.io/client-go/tools/cache" ) +// TestEventProcessorExit is expected to timeout if the event processor fails +// to exit when stopped. +func TestEventProcessorExit(t *testing.T) { + event := watch.Event{} + + tests := []struct { + name string + write func(e *eventProcessor) + }{ + { + name: "exit on blocked read", + write: func(e *eventProcessor) { + e.push(event) + }, + }, + { + name: "exit on blocked write", + write: func(e *eventProcessor) { + e.push(event) + e.push(event) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + out := make(chan watch.Event) + e := newEventProcessor(out) + + test.write(e) + + exited := make(chan struct{}) + go func() { + e.run() + close(exited) + }() + + <-out + e.stop() + goruntime.Gosched() + <-exited + }) + } +} + +type apiInt int + +func (apiInt) GetObjectKind() schema.ObjectKind { return nil } +func (apiInt) DeepCopyObject() runtime.Object { return nil } + +func TestEventProcessorOrdersEvents(t *testing.T) { + out := make(chan watch.Event) + e := newEventProcessor(out) + go e.run() + + numProcessed := 0 + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + go func() { + for i := 0; i < 1000; i++ { + e := <-out + if got, want := int(e.Object.(apiInt)), i; got != want { + t.Errorf("unexpected event: got=%d, want=%d", got, want) + } + numProcessed++ + } + cancel() + }() + + for i := 0; i < 1000; i++ { + e.push(watch.Event{Object: apiInt(i)}) + } + + <-ctx.Done() + e.stop() + + if numProcessed != 1000 { + t.Errorf("unexpected number of events processed: %d", numProcessed) + } + +} + type byEventTypeAndName []watch.Event func (a byEventTypeAndName) Len() int { return len(a) } @@ -51,44 +133,6 @@ func (a byEventTypeAndName) Less(i, j int) bool { return a[i].Object.(*corev1.Secret).Name < a[j].Object.(*corev1.Secret).Name } -func TestTicketer(t *testing.T) { - tg := newTicketer() - - const numTickets = 100 // current golang limit for race detector is 8192 simultaneously alive goroutines - var tickets []uint64 - for i := 0; i < numTickets; i++ { - ticket := tg.GetTicket() - tickets = append(tickets, ticket) - - exp, got := uint64(i), ticket - if got != exp { - t.Fatalf("expected ticket %d, got %d", exp, got) - } - } - - // shuffle tickets - rand.Shuffle(len(tickets), func(i, j int) { - tickets[i], tickets[j] = tickets[j], tickets[i] - }) - - res := make(chan uint64, len(tickets)) - for _, ticket := range tickets { - go func(ticket uint64) { - time.Sleep(time.Duration(rand.Intn(50)) * time.Millisecond) - tg.WaitForTicket(ticket, func() { - res <- ticket - }) - }(ticket) - } - - for i := 0; i < numTickets; i++ { - exp, got := uint64(i), <-res - if got != exp { - t.Fatalf("expected ticket %d, got %d", exp, got) - } - } -} - func TestNewInformerWatcher(t *testing.T) { // Make sure there are no 2 same types of events on a secret with the same name or that might be flaky. tt := []struct { diff --git a/staging/src/k8s.io/cloud-provider/go.mod b/staging/src/k8s.io/cloud-provider/go.mod index ff46260f0a3..cb3a7b14966 100644 --- a/staging/src/k8s.io/cloud-provider/go.mod +++ b/staging/src/k8s.io/cloud-provider/go.mod @@ -8,7 +8,7 @@ require ( k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 k8s.io/client-go v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/utils v0.0.0-20190221042446-c2654d5206da ) diff --git a/staging/src/k8s.io/cloud-provider/go.sum b/staging/src/k8s.io/cloud-provider/go.sum index 72bcc2914e3..d17d7d58079 100644 --- a/staging/src/k8s.io/cloud-provider/go.sum +++ b/staging/src/k8s.io/cloud-provider/go.sum @@ -79,8 +79,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/cluster-bootstrap/go.sum b/staging/src/k8s.io/cluster-bootstrap/go.sum index 44ad19d7d2f..53da16484ca 100644 --- a/staging/src/k8s.io/cluster-bootstrap/go.sum +++ b/staging/src/k8s.io/cluster-bootstrap/go.sum @@ -48,8 +48,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/staging/src/k8s.io/code-generator/go.mod b/staging/src/k8s.io/code-generator/go.mod index 37b7b16e75c..7f46b175177 100644 --- a/staging/src/k8s.io/code-generator/go.mod +++ b/staging/src/k8s.io/code-generator/go.mod @@ -11,7 +11,7 @@ require ( gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e // indirect k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 ) replace ( diff --git a/staging/src/k8s.io/code-generator/go.sum b/staging/src/k8s.io/code-generator/go.sum index f40a1f4059f..c9cf7c62148 100644 --- a/staging/src/k8s.io/code-generator/go.sum +++ b/staging/src/k8s.io/code-generator/go.sum @@ -20,8 +20,8 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGB gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= diff --git a/staging/src/k8s.io/component-base/go.mod b/staging/src/k8s.io/component-base/go.mod index d72e130ad12..a5e2487367a 100644 --- a/staging/src/k8s.io/component-base/go.mod +++ b/staging/src/k8s.io/component-base/go.mod @@ -12,7 +12,7 @@ require ( github.com/spf13/pflag v1.0.1 github.com/stretchr/testify v1.2.2 k8s.io/apimachinery v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/utils v0.0.0-20190221042446-c2654d5206da ) diff --git a/staging/src/k8s.io/component-base/go.sum b/staging/src/k8s.io/component-base/go.sum index 4ef3cbf6e3e..2632bc99222 100644 --- a/staging/src/k8s.io/component-base/go.sum +++ b/staging/src/k8s.io/component-base/go.sum @@ -65,8 +65,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= diff --git a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go index f93bf5c6803..4b410c5d7c5 100644 --- a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go +++ b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go @@ -1676,6 +1676,8 @@ type WindowsContainerSecurityContext struct { // exist in the container image and be resolved there by the runtime; // otherwise, the runtime MUST return error. RunAsUsername string `protobuf:"bytes,1,opt,name=run_as_username,json=runAsUsername,proto3" json:"run_as_username,omitempty"` + // The contents of the GMSA credential spec to use to run this container. + CredentialSpec string `protobuf:"bytes,2,opt,name=credential_spec,json=credentialSpec,proto3" json:"credential_spec,omitempty"` } func (m *WindowsContainerSecurityContext) Reset() { *m = WindowsContainerSecurityContext{} } @@ -1691,6 +1693,13 @@ func (m *WindowsContainerSecurityContext) GetRunAsUsername() string { return "" } +func (m *WindowsContainerSecurityContext) GetCredentialSpec() string { + if m != nil { + return m.CredentialSpec + } + return "" +} + // WindowsContainerConfig contains platform-specific configuration for // Windows-based containers. type WindowsContainerConfig struct { @@ -6485,6 +6494,12 @@ func (m *WindowsContainerSecurityContext) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintApi(dAtA, i, uint64(len(m.RunAsUsername))) i += copy(dAtA[i:], m.RunAsUsername) } + if len(m.CredentialSpec) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.CredentialSpec))) + i += copy(dAtA[i:], m.CredentialSpec) + } return i, nil } @@ -9775,6 +9790,10 @@ func (m *WindowsContainerSecurityContext) Size() (n int) { if l > 0 { n += 1 + l + sovApi(uint64(l)) } + l = len(m.CredentialSpec) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } return n } @@ -11381,6 +11400,7 @@ func (this *WindowsContainerSecurityContext) String() string { } s := strings.Join([]string{`&WindowsContainerSecurityContext{`, `RunAsUsername:` + fmt.Sprintf("%v", this.RunAsUsername) + `,`, + `CredentialSpec:` + fmt.Sprintf("%v", this.CredentialSpec) + `,`, `}`, }, "") return s @@ -17980,6 +18000,35 @@ func (m *WindowsContainerSecurityContext) Unmarshal(dAtA []byte) error { } m.RunAsUsername = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CredentialSpec", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CredentialSpec = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipApi(dAtA[iNdEx:]) @@ -26960,300 +27009,302 @@ var ( func init() { proto.RegisterFile("api.proto", fileDescriptorApi) } var fileDescriptorApi = []byte{ - // 4720 bytes of a gzipped FileDescriptorProto + // 4737 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5c, 0xcd, 0x6f, 0x1b, 0x49, - 0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x89, 0x2a, 0xcb, 0x16, 0x4d, 0x8f, 0x35, 0x56, 0xcf, - 0xf8, 0x73, 0xc7, 0xf2, 0x58, 0xb3, 0xeb, 0x89, 0xed, 0x59, 0xdb, 0xb4, 0x24, 0xdb, 0xcc, 0xda, - 0x14, 0xd3, 0x94, 0xe6, 0x63, 0x67, 0x80, 0xde, 0x16, 0xbb, 0x44, 0xf5, 0x9a, 0xec, 0xea, 0xe9, + 0x76, 0x57, 0x93, 0xfa, 0x20, 0x1f, 0x45, 0x8a, 0x2a, 0xcb, 0x16, 0x4d, 0x8f, 0x35, 0x56, 0xcf, + 0xf8, 0x73, 0xc7, 0xf2, 0x58, 0xb3, 0xeb, 0x89, 0xed, 0x59, 0xdb, 0x34, 0x25, 0xdb, 0xcc, 0xda, + 0x14, 0xd3, 0x94, 0xe6, 0x63, 0x67, 0x80, 0xde, 0x16, 0xbb, 0x44, 0xf5, 0x9a, 0xec, 0xee, 0xe9, 0x6e, 0xda, 0x56, 0x02, 0x04, 0x0b, 0x2c, 0xb2, 0x87, 0x00, 0x01, 0x72, 0xce, 0x71, 0x73, 0xc8, 0x21, 0xb7, 0x00, 0x41, 0x0e, 0x39, 0x6d, 0x90, 0xc3, 0x5e, 0x02, 0xe4, 0xb4, 0x48, 0x90, 0x4b, - 0x66, 0x92, 0x5c, 0x02, 0x24, 0xc8, 0x1f, 0x90, 0x43, 0x50, 0x5f, 0xcd, 0xfe, 0xe4, 0x87, 0xc7, - 0xbb, 0xb3, 0x39, 0xa9, 0xfb, 0xf5, 0x7b, 0xaf, 0x5e, 0xbd, 0x7a, 0xf5, 0xea, 0xd5, 0xaf, 0x8a, - 0x82, 0xa2, 0xe1, 0x58, 0x9b, 0x8e, 0x4b, 0x7c, 0x82, 0x2a, 0xee, 0xc0, 0xf6, 0xad, 0x3e, 0xde, - 0x7c, 0x71, 0xd3, 0xe8, 0x39, 0xc7, 0xc6, 0x56, 0xed, 0x7a, 0xd7, 0xf2, 0x8f, 0x07, 0x87, 0x9b, - 0x1d, 0xd2, 0xbf, 0xd1, 0x25, 0x5d, 0x72, 0x83, 0x31, 0x1e, 0x0e, 0x8e, 0xd8, 0x1b, 0x7b, 0x61, - 0x4f, 0x5c, 0x81, 0x7a, 0x0d, 0x96, 0x3e, 0xc6, 0xae, 0x67, 0x11, 0x5b, 0xc3, 0x5f, 0x0e, 0xb0, - 0xe7, 0xa3, 0x2a, 0x2c, 0xbc, 0xe0, 0x94, 0xaa, 0x72, 0x41, 0xb9, 0x52, 0xd4, 0xe4, 0xab, 0xfa, - 0x17, 0x0a, 0x2c, 0x07, 0xcc, 0x9e, 0x43, 0x6c, 0x0f, 0x67, 0x73, 0xa3, 0x0d, 0x58, 0x14, 0xc6, - 0xe9, 0xb6, 0xd1, 0xc7, 0xd5, 0x1c, 0xfb, 0x5c, 0x12, 0xb4, 0xa6, 0xd1, 0xc7, 0xe8, 0x32, 0x2c, - 0x4b, 0x16, 0xa9, 0x24, 0xcf, 0xb8, 0x96, 0x04, 0x59, 0xb4, 0x86, 0x36, 0xe1, 0x94, 0x64, 0x34, - 0x1c, 0x2b, 0x60, 0x9e, 0x65, 0xcc, 0x2b, 0xe2, 0x53, 0xdd, 0xb1, 0x04, 0xbf, 0xfa, 0x39, 0x14, - 0x77, 0x9a, 0xed, 0x6d, 0x62, 0x1f, 0x59, 0x5d, 0x6a, 0xa2, 0x87, 0x5d, 0x2a, 0x53, 0x55, 0x2e, - 0xe4, 0xa9, 0x89, 0xe2, 0x15, 0xd5, 0xa0, 0xe0, 0x61, 0xc3, 0xed, 0x1c, 0x63, 0xaf, 0x9a, 0x63, - 0x9f, 0x82, 0x77, 0x2a, 0x45, 0x1c, 0xdf, 0x22, 0xb6, 0x57, 0xcd, 0x73, 0x29, 0xf1, 0xaa, 0xfe, - 0x5c, 0x81, 0x52, 0x8b, 0xb8, 0xfe, 0x33, 0xc3, 0x71, 0x2c, 0xbb, 0x8b, 0x6e, 0x41, 0x81, 0xf9, - 0xb2, 0x43, 0x7a, 0xcc, 0x07, 0x4b, 0x5b, 0xb5, 0xcd, 0xf8, 0xb0, 0x6c, 0xb6, 0x04, 0x87, 0x16, - 0xf0, 0xa2, 0x8b, 0xb0, 0xd4, 0x21, 0xb6, 0x6f, 0x58, 0x36, 0x76, 0x75, 0x87, 0xb8, 0x3e, 0x73, - 0xd1, 0x9c, 0x56, 0x0e, 0xa8, 0xb4, 0x15, 0x74, 0x0e, 0x8a, 0xc7, 0xc4, 0xf3, 0x39, 0x47, 0x9e, - 0x71, 0x14, 0x28, 0x81, 0x7d, 0x5c, 0x83, 0x05, 0xf6, 0xd1, 0x72, 0x84, 0x33, 0xe6, 0xe9, 0x6b, - 0xc3, 0x51, 0x7f, 0xa5, 0xc0, 0xdc, 0x33, 0x32, 0xb0, 0xfd, 0x58, 0x33, 0x86, 0x7f, 0x2c, 0x06, - 0x2a, 0xd4, 0x8c, 0xe1, 0x1f, 0x0f, 0x9b, 0xa1, 0x1c, 0x7c, 0xac, 0x78, 0x33, 0xf4, 0x63, 0x0d, - 0x0a, 0x2e, 0x36, 0x4c, 0x62, 0xf7, 0x4e, 0x98, 0x09, 0x05, 0x2d, 0x78, 0xa7, 0x83, 0xe8, 0xe1, - 0x9e, 0x65, 0x0f, 0x5e, 0xe9, 0x2e, 0xee, 0x19, 0x87, 0xb8, 0xc7, 0x4c, 0x29, 0x68, 0x4b, 0x82, - 0xac, 0x71, 0x2a, 0xda, 0x81, 0x92, 0xe3, 0x12, 0xc7, 0xe8, 0x1a, 0xd4, 0x8f, 0xd5, 0x39, 0xe6, - 0x2a, 0x35, 0xe9, 0x2a, 0x66, 0x76, 0x6b, 0xc8, 0xa9, 0x85, 0xc5, 0xd4, 0xbf, 0x52, 0x60, 0x99, - 0x06, 0x8f, 0xe7, 0x18, 0x1d, 0xbc, 0xc7, 0x86, 0x04, 0xdd, 0x86, 0x05, 0x1b, 0xfb, 0x2f, 0x89, - 0xfb, 0x5c, 0x0c, 0xc0, 0xdb, 0x49, 0xad, 0x81, 0xcc, 0x33, 0x62, 0x62, 0x4d, 0xf2, 0xa3, 0x9b, - 0x90, 0x77, 0x2c, 0x93, 0x75, 0x78, 0x02, 0x31, 0xca, 0x4b, 0x45, 0x2c, 0xa7, 0xc3, 0xfc, 0x30, - 0x89, 0x88, 0xe5, 0x74, 0x54, 0x15, 0xa0, 0x61, 0xfb, 0xb7, 0xbe, 0xfb, 0xb1, 0xd1, 0x1b, 0x60, - 0xb4, 0x0a, 0x73, 0x2f, 0xe8, 0x03, 0x33, 0x36, 0xaf, 0xf1, 0x17, 0xf5, 0xab, 0x3c, 0x9c, 0x7b, - 0x4a, 0xfd, 0xd5, 0x36, 0x6c, 0xf3, 0x90, 0xbc, 0x6a, 0xe3, 0xce, 0xc0, 0xb5, 0xfc, 0x93, 0x6d, - 0x62, 0xfb, 0xf8, 0x95, 0x8f, 0x9a, 0xb0, 0x62, 0x4b, 0xcd, 0xba, 0x0c, 0x4d, 0xaa, 0xa1, 0xb4, - 0xb5, 0x31, 0xc2, 0x08, 0xee, 0x22, 0xad, 0x62, 0x47, 0x09, 0x1e, 0x7a, 0x32, 0x1c, 0x37, 0xa9, - 0x2d, 0xc7, 0xb4, 0xa5, 0x74, 0xa9, 0xbd, 0xcb, 0x2c, 0x13, 0xba, 0xe4, 0xc0, 0x4a, 0x4d, 0x1f, - 0x01, 0x9d, 0xd5, 0xba, 0xe1, 0xe9, 0x03, 0x0f, 0xbb, 0xcc, 0x31, 0xa5, 0xad, 0xb7, 0x92, 0x5a, - 0x86, 0x2e, 0xd0, 0x8a, 0xee, 0xc0, 0xae, 0x7b, 0x07, 0x1e, 0x76, 0x59, 0x12, 0x10, 0xb1, 0xa4, - 0xbb, 0x84, 0xf8, 0x47, 0x9e, 0x8c, 0x1f, 0x49, 0xd6, 0x18, 0x15, 0xdd, 0x80, 0x53, 0xde, 0xc0, - 0x71, 0x7a, 0xb8, 0x8f, 0x6d, 0xdf, 0xe8, 0xe9, 0x5d, 0x97, 0x0c, 0x1c, 0xaf, 0x3a, 0x77, 0x21, - 0x7f, 0x25, 0xaf, 0xa1, 0xf0, 0xa7, 0xc7, 0xec, 0x0b, 0x5a, 0x07, 0x70, 0x5c, 0xeb, 0x85, 0xd5, - 0xc3, 0x5d, 0x6c, 0x56, 0xe7, 0x99, 0xd2, 0x10, 0x05, 0xbd, 0x0f, 0xab, 0x1e, 0xee, 0x74, 0x48, - 0xdf, 0xd1, 0x1d, 0x97, 0x1c, 0x59, 0x3d, 0xcc, 0xa3, 0x7f, 0x81, 0x45, 0x3f, 0x12, 0xdf, 0x5a, - 0xfc, 0x13, 0x9b, 0x07, 0xf7, 0x58, 0x4e, 0xa3, 0x3d, 0x65, 0x8d, 0x57, 0x0b, 0x13, 0x74, 0x15, - 0x58, 0x57, 0x99, 0x49, 0xea, 0xcf, 0x73, 0x70, 0x9a, 0x79, 0xb2, 0x45, 0x4c, 0x31, 0xcc, 0x22, - 0x49, 0xbd, 0x03, 0xe5, 0x0e, 0xd3, 0xa9, 0x3b, 0x86, 0x8b, 0x6d, 0x5f, 0x4c, 0xd2, 0x45, 0x4e, - 0x6c, 0x31, 0x1a, 0xfa, 0x14, 0x2a, 0x9e, 0x88, 0x0a, 0xbd, 0xc3, 0xc3, 0x42, 0x8c, 0xd9, 0xf5, - 0xa4, 0x09, 0x23, 0x62, 0x49, 0x5b, 0xf6, 0x12, 0xc1, 0xb5, 0xe0, 0x9d, 0x78, 0x1d, 0xbf, 0xc7, - 0xb3, 0x5d, 0x69, 0xeb, 0xbb, 0x19, 0x0a, 0xe3, 0x86, 0x6f, 0xb6, 0xb9, 0xd8, 0xae, 0xed, 0xbb, - 0x27, 0x9a, 0x54, 0x52, 0xbb, 0x03, 0x8b, 0xe1, 0x0f, 0xa8, 0x02, 0xf9, 0xe7, 0xf8, 0x44, 0x74, - 0x8a, 0x3e, 0x0e, 0x27, 0x01, 0xcf, 0x35, 0xfc, 0xe5, 0x4e, 0xee, 0x77, 0x14, 0xd5, 0x05, 0x34, - 0x6c, 0xe5, 0x19, 0xf6, 0x0d, 0xd3, 0xf0, 0x0d, 0x84, 0x60, 0x96, 0x2d, 0x23, 0x5c, 0x05, 0x7b, - 0xa6, 0x5a, 0x07, 0x62, 0xf2, 0x16, 0x35, 0xfa, 0x88, 0xde, 0x82, 0x62, 0x10, 0xe8, 0x62, 0x2d, - 0x19, 0x12, 0x68, 0x4e, 0x37, 0x7c, 0x1f, 0xf7, 0x1d, 0x9f, 0x85, 0x58, 0x59, 0x93, 0xaf, 0xea, - 0x7f, 0xcf, 0x42, 0x25, 0x31, 0x26, 0x0f, 0xa0, 0xd0, 0x17, 0xcd, 0x8b, 0x89, 0xf6, 0x6e, 0x4a, - 0x62, 0x4f, 0x98, 0xaa, 0x05, 0x52, 0x34, 0x6f, 0xd2, 0x1c, 0x1a, 0x5a, 0xff, 0x82, 0x77, 0x3a, - 0xe2, 0x3d, 0xd2, 0xd5, 0x4d, 0xcb, 0xc5, 0x1d, 0x9f, 0xb8, 0x27, 0xc2, 0xdc, 0xc5, 0x1e, 0xe9, - 0xee, 0x48, 0x1a, 0xba, 0x03, 0x60, 0xda, 0x1e, 0x1d, 0xec, 0x23, 0xab, 0xcb, 0x8c, 0x2e, 0x6d, - 0x9d, 0x4b, 0x1a, 0x11, 0x2c, 0x76, 0x5a, 0xd1, 0xb4, 0x3d, 0x61, 0xfe, 0x43, 0x28, 0xd3, 0x35, - 0x43, 0xef, 0xf3, 0x75, 0x8a, 0xcf, 0x94, 0xd2, 0xd6, 0xf9, 0xb4, 0x3e, 0x04, 0xab, 0x99, 0xb6, - 0xe8, 0x0c, 0x5f, 0x3c, 0xf4, 0x08, 0xe6, 0x59, 0xf2, 0xf6, 0xaa, 0xf3, 0x4c, 0x78, 0x73, 0x94, - 0x03, 0x44, 0x44, 0x3c, 0x65, 0x02, 0x3c, 0x20, 0x84, 0x34, 0x3a, 0x80, 0x92, 0x61, 0xdb, 0xc4, - 0x37, 0x78, 0xa2, 0x59, 0x60, 0xca, 0x3e, 0x98, 0x40, 0x59, 0x7d, 0x28, 0xc5, 0x35, 0x86, 0xf5, - 0xa0, 0xef, 0xc3, 0x1c, 0xcb, 0x44, 0x62, 0x22, 0x5e, 0x9e, 0x30, 0x68, 0x35, 0x2e, 0x55, 0xbb, - 0x0d, 0xa5, 0x90, 0xb1, 0xd3, 0x04, 0x69, 0xed, 0x1e, 0x54, 0xe2, 0xa6, 0x4d, 0x15, 0xe4, 0x7f, - 0x00, 0xab, 0xda, 0xc0, 0x1e, 0x1a, 0x26, 0xab, 0xaf, 0x3b, 0x30, 0x2f, 0x06, 0x9b, 0x47, 0x9c, - 0x3a, 0xde, 0x47, 0x9a, 0x90, 0x08, 0x97, 0x53, 0xc7, 0x86, 0x6d, 0xf6, 0xb0, 0x2b, 0xda, 0x95, - 0xe5, 0xd4, 0x13, 0x4e, 0x55, 0xbf, 0x0f, 0xa7, 0x63, 0x8d, 0x8b, 0x6a, 0xee, 0x5d, 0x58, 0x72, - 0x88, 0xa9, 0x7b, 0x9c, 0xac, 0x5b, 0xa6, 0x4c, 0x43, 0x4e, 0xc0, 0xdb, 0x30, 0xa9, 0x78, 0xdb, - 0x27, 0x4e, 0xd2, 0xf8, 0xc9, 0xc4, 0xab, 0x70, 0x26, 0x2e, 0xce, 0x9b, 0x57, 0xef, 0xc3, 0x9a, - 0x86, 0xfb, 0xe4, 0x05, 0x7e, 0x5d, 0xd5, 0x35, 0xa8, 0x26, 0x15, 0x08, 0xe5, 0x9f, 0xc1, 0xda, - 0x90, 0xda, 0xf6, 0x0d, 0x7f, 0xe0, 0x4d, 0xa5, 0x5c, 0x94, 0xba, 0x87, 0xc4, 0xe3, 0xc3, 0x59, - 0xd0, 0xe4, 0xab, 0x7a, 0x35, 0xac, 0xba, 0xc9, 0x2b, 0x0b, 0xde, 0x02, 0x5a, 0x82, 0x9c, 0xe5, - 0x08, 0x75, 0x39, 0xcb, 0x51, 0x9f, 0x40, 0x31, 0x58, 0x9a, 0xd1, 0xdd, 0x61, 0x8d, 0x99, 0x9b, - 0x74, 0x21, 0x0f, 0xca, 0xd0, 0xfd, 0xc4, 0x52, 0x22, 0x9a, 0xbc, 0x0b, 0x10, 0xa4, 0x3c, 0x59, - 0x21, 0x9c, 0x1b, 0xa1, 0x58, 0x0b, 0xb1, 0xab, 0x3f, 0x9d, 0x0b, 0x27, 0xc2, 0x50, 0x27, 0xcc, - 0xa0, 0x13, 0x66, 0x24, 0x31, 0xe6, 0x5e, 0x2b, 0x31, 0x7e, 0x08, 0x73, 0x9e, 0x6f, 0xf8, 0x58, - 0x54, 0x51, 0x1b, 0xa3, 0xc4, 0xa9, 0x11, 0x58, 0xe3, 0xfc, 0xe8, 0x3c, 0x40, 0xc7, 0xc5, 0x86, - 0x8f, 0x4d, 0xdd, 0xe0, 0x59, 0x3c, 0xaf, 0x15, 0x05, 0xa5, 0xee, 0xa3, 0xed, 0x61, 0x25, 0x38, - 0xc7, 0x0c, 0xbb, 0x3a, 0x4a, 0x73, 0x64, 0xa8, 0x86, 0x35, 0x61, 0x90, 0x55, 0xe6, 0x27, 0xcc, - 0x2a, 0x42, 0x01, 0x97, 0x0a, 0xe5, 0xcc, 0x85, 0xf1, 0x39, 0x93, 0x8b, 0x4e, 0x92, 0x33, 0x0b, - 0xe3, 0x73, 0xa6, 0x50, 0x36, 0x3a, 0x67, 0xa6, 0x64, 0x89, 0x62, 0x5a, 0x96, 0xf8, 0x36, 0xb3, - 0xe3, 0x3f, 0x2b, 0x50, 0x4d, 0x4e, 0x56, 0x91, 0xa4, 0xee, 0xc0, 0xbc, 0xc7, 0x28, 0x93, 0xa4, - 0x48, 0x21, 0x2b, 0x24, 0xd0, 0x13, 0x98, 0xb5, 0xec, 0x23, 0xc2, 0x76, 0x7b, 0xa9, 0x45, 0x4e, - 0x56, 0xab, 0x9b, 0x0d, 0xfb, 0x88, 0x70, 0x6f, 0x32, 0x0d, 0xb5, 0x0f, 0xa1, 0x18, 0x90, 0xa6, - 0xea, 0xdb, 0x1e, 0xac, 0xc6, 0x62, 0x9b, 0xef, 0x0a, 0x82, 0x29, 0xa1, 0x4c, 0x37, 0x25, 0xd4, - 0x9f, 0xe4, 0xc2, 0x53, 0xf6, 0x91, 0xd5, 0xf3, 0xb1, 0x9b, 0x98, 0xb2, 0x1f, 0x49, 0xed, 0x7c, - 0xbe, 0x5e, 0x1a, 0xab, 0x9d, 0x17, 0xaf, 0x62, 0xd6, 0x7d, 0x01, 0x4b, 0x2c, 0x28, 0x75, 0x0f, - 0xf7, 0x58, 0x65, 0x22, 0xaa, 0xc4, 0xef, 0x8d, 0x52, 0xc3, 0x2d, 0xe1, 0xa1, 0xdd, 0x16, 0x72, - 0xdc, 0x83, 0xe5, 0x5e, 0x98, 0x56, 0x7b, 0x00, 0x28, 0xc9, 0x34, 0x95, 0x4f, 0xdb, 0x34, 0x17, - 0xd2, 0x2d, 0x71, 0xca, 0x72, 0x7a, 0xc4, 0xcc, 0x98, 0x24, 0x56, 0xb8, 0xc1, 0x9a, 0x90, 0x50, - 0xff, 0x2b, 0x0f, 0x30, 0xfc, 0xf8, 0xff, 0x28, 0x09, 0x3e, 0x08, 0x12, 0x10, 0xaf, 0xf8, 0xae, - 0x8c, 0x52, 0x9c, 0x9a, 0x7a, 0xf6, 0xa2, 0xa9, 0x87, 0xd7, 0x7e, 0xd7, 0x47, 0xaa, 0x99, 0x3a, - 0xe9, 0x2c, 0xfc, 0xb6, 0x25, 0x9d, 0xa7, 0x70, 0x26, 0x1e, 0x44, 0x22, 0xe3, 0x6c, 0xc1, 0x9c, - 0xe5, 0xe3, 0x3e, 0xc7, 0x8f, 0x52, 0xf7, 0x7b, 0x21, 0x21, 0xce, 0xaa, 0x6e, 0x40, 0xb1, 0xd1, - 0x37, 0xba, 0xb8, 0xed, 0xe0, 0x0e, 0x6d, 0xd4, 0xa2, 0x2f, 0xc2, 0x10, 0xfe, 0xa2, 0x6e, 0x41, - 0xe1, 0x07, 0xf8, 0x84, 0xcf, 0xfe, 0x09, 0x0d, 0x55, 0xff, 0x24, 0x07, 0x6b, 0x6c, 0xf5, 0xd9, - 0x96, 0xe8, 0x8d, 0x86, 0x3d, 0x32, 0x70, 0x3b, 0xd8, 0x63, 0x61, 0xe1, 0x0c, 0x74, 0x07, 0xbb, - 0x16, 0x31, 0x05, 0xb8, 0x50, 0xec, 0x38, 0x83, 0x16, 0x23, 0xa0, 0x73, 0x40, 0x5f, 0xf4, 0x2f, - 0x07, 0x44, 0x44, 0x6c, 0x5e, 0x2b, 0x74, 0x9c, 0xc1, 0xef, 0xd1, 0x77, 0x29, 0xeb, 0x1d, 0x1b, - 0x2e, 0xf6, 0x58, 0x40, 0x72, 0xd9, 0x36, 0x23, 0xa0, 0x9b, 0x70, 0xba, 0x8f, 0xfb, 0xc4, 0x3d, - 0xd1, 0x7b, 0x56, 0xdf, 0xf2, 0x75, 0xcb, 0xd6, 0x0f, 0x4f, 0x7c, 0xec, 0x89, 0xe0, 0x43, 0xfc, - 0xe3, 0x53, 0xfa, 0xad, 0x61, 0x3f, 0xa4, 0x5f, 0x90, 0x0a, 0x65, 0x42, 0xfa, 0xba, 0xd7, 0x21, - 0x2e, 0xd6, 0x0d, 0xf3, 0xc7, 0x6c, 0x41, 0xce, 0x6b, 0x25, 0x42, 0xfa, 0x6d, 0x4a, 0xab, 0x9b, - 0x3f, 0x46, 0x6f, 0x43, 0xa9, 0xe3, 0x0c, 0x3c, 0xec, 0xeb, 0xf4, 0x0f, 0x5b, 0x6f, 0x8b, 0x1a, - 0x70, 0xd2, 0xb6, 0x33, 0xf0, 0x42, 0x0c, 0x7d, 0xea, 0xff, 0x85, 0x30, 0xc3, 0x33, 0xea, 0x66, - 0x03, 0xca, 0x11, 0x70, 0x82, 0xee, 0x13, 0x19, 0x0a, 0x21, 0xf6, 0x89, 0xf4, 0x99, 0xd2, 0x5c, - 0xd2, 0x93, 0x9e, 0x64, 0xcf, 0x94, 0xe6, 0x9f, 0x38, 0x72, 0x93, 0xc8, 0x9e, 0xa9, 0xcb, 0x7b, - 0xf8, 0x85, 0x00, 0xb0, 0x8a, 0x1a, 0x7f, 0x51, 0x4d, 0x80, 0x6d, 0xc3, 0x31, 0x0e, 0xad, 0x9e, - 0xe5, 0x9f, 0xa0, 0xab, 0x50, 0x31, 0x4c, 0x53, 0xef, 0x48, 0x8a, 0x85, 0x25, 0xac, 0xb8, 0x6c, - 0x98, 0xe6, 0x76, 0x88, 0x8c, 0xbe, 0x03, 0x2b, 0xa6, 0x4b, 0x9c, 0x28, 0x2f, 0xc7, 0x19, 0x2b, - 0xf4, 0x43, 0x98, 0x59, 0xfd, 0xf7, 0x39, 0x38, 0x1f, 0x1d, 0xd8, 0x38, 0x00, 0xf4, 0x00, 0x16, - 0x63, 0xad, 0x66, 0x80, 0x0f, 0x43, 0x6b, 0xb5, 0x88, 0x44, 0x0c, 0x10, 0xc9, 0x25, 0x00, 0x91, - 0x54, 0x88, 0x29, 0xff, 0x46, 0x21, 0xa6, 0xd9, 0x37, 0x02, 0x31, 0xcd, 0x4d, 0x07, 0x31, 0x5d, - 0x62, 0xd9, 0x47, 0x4a, 0xb3, 0xdd, 0x38, 0x0f, 0xb5, 0x72, 0xc0, 0x63, 0x4b, 0x3c, 0x3a, 0x06, - 0x45, 0x2d, 0x4c, 0x03, 0x45, 0x15, 0x32, 0xa1, 0x28, 0x1a, 0x35, 0x8e, 0x63, 0xb8, 0x7d, 0xe2, - 0x4a, 0xac, 0x49, 0x54, 0x5d, 0xcb, 0x92, 0x2e, 0x70, 0xa6, 0x4c, 0x54, 0x0a, 0x32, 0x51, 0xa9, - 0x0b, 0xb0, 0x68, 0x13, 0xdd, 0xc6, 0x2f, 0x75, 0x3a, 0x96, 0x5e, 0xb5, 0xc4, 0x07, 0xd6, 0x26, - 0x4d, 0xfc, 0xb2, 0x45, 0x29, 0x09, 0xdc, 0x6a, 0x71, 0x3a, 0xdc, 0x0a, 0x6d, 0xc0, 0x62, 0xdf, - 0xf0, 0x9e, 0x63, 0x93, 0x99, 0xe2, 0x55, 0xcb, 0x2c, 0x88, 0x4b, 0x9c, 0x46, 0x6d, 0xf0, 0xd0, - 0x45, 0x08, 0x9c, 0x24, 0x98, 0x96, 0x18, 0x53, 0x59, 0x52, 0x19, 0x9b, 0xfa, 0xb7, 0x0a, 0xac, - 0x46, 0xc3, 0x5c, 0xa0, 0x15, 0x8f, 0xa1, 0xe8, 0xca, 0x4c, 0x26, 0x42, 0xfb, 0x6a, 0x46, 0xe1, - 0x9d, 0x4c, 0x7d, 0xda, 0x50, 0x16, 0xfd, 0x30, 0x13, 0x24, 0xbb, 0x31, 0x4e, 0xdf, 0x38, 0x98, - 0x4c, 0x6d, 0xc0, 0xdb, 0x9f, 0x58, 0xb6, 0x49, 0x5e, 0x7a, 0x99, 0xb3, 0x34, 0x25, 0xd6, 0x94, - 0x94, 0x58, 0x53, 0x7f, 0xa1, 0xc0, 0x99, 0xb8, 0x2e, 0xe1, 0x8a, 0x46, 0xd2, 0x15, 0xdf, 0x49, - 0x9a, 0x1e, 0x17, 0x4e, 0x75, 0xc6, 0x17, 0x99, 0xce, 0xb8, 0x39, 0x5e, 0xe3, 0x58, 0x77, 0xfc, - 0xa5, 0x02, 0x67, 0x33, 0xcd, 0x88, 0x2d, 0x29, 0x4a, 0x7c, 0x49, 0x11, 0xcb, 0x51, 0x87, 0x0c, - 0x6c, 0x3f, 0xb4, 0x1c, 0x6d, 0xb3, 0x43, 0x0b, 0x9e, 0xf7, 0xf5, 0xbe, 0xf1, 0xca, 0xea, 0x0f, - 0xfa, 0x62, 0x3d, 0xa2, 0xea, 0x9e, 0x71, 0xca, 0x6b, 0x2c, 0x48, 0x6a, 0x1d, 0x56, 0x02, 0x2b, - 0x47, 0xc2, 0x8a, 0x21, 0x98, 0x30, 0x17, 0x85, 0x09, 0x6d, 0x98, 0xdf, 0xc1, 0x2f, 0xac, 0x0e, - 0x7e, 0x23, 0xa7, 0x2a, 0x17, 0xa0, 0xe4, 0x60, 0xb7, 0x6f, 0x79, 0x5e, 0x90, 0x68, 0x8b, 0x5a, - 0x98, 0xa4, 0xfe, 0xc7, 0x3c, 0x2c, 0xc7, 0xa3, 0xe3, 0x7e, 0x02, 0x95, 0x7c, 0x27, 0x65, 0x09, - 0x88, 0x77, 0x34, 0x54, 0x76, 0xde, 0x94, 0xc5, 0x48, 0x2e, 0x0b, 0x1a, 0x08, 0x0a, 0x17, 0x51, - 0xa9, 0x50, 0x8f, 0x74, 0x48, 0xbf, 0x6f, 0xd8, 0xa6, 0x3c, 0x0c, 0x13, 0xaf, 0xd4, 0x7f, 0x86, - 0xdb, 0xa5, 0x6e, 0xa7, 0x64, 0xf6, 0x4c, 0x07, 0x8f, 0xee, 0xa3, 0x2d, 0x9b, 0xa1, 0x9b, 0x2c, - 0x59, 0x17, 0x35, 0x10, 0xa4, 0x1d, 0xcb, 0x45, 0x9b, 0x30, 0x8b, 0xed, 0x17, 0xb2, 0xae, 0x4c, - 0x39, 0x2d, 0x93, 0x65, 0x91, 0xc6, 0xf8, 0xd0, 0x0d, 0x98, 0xef, 0xd3, 0xb0, 0x90, 0x3b, 0xea, - 0xb5, 0x8c, 0x43, 0x23, 0x4d, 0xb0, 0xa1, 0x2d, 0x58, 0x30, 0xd9, 0x38, 0xc9, 0x6d, 0x73, 0x35, - 0x05, 0x33, 0x65, 0x0c, 0x9a, 0x64, 0x44, 0xbb, 0x41, 0xd5, 0x5c, 0xcc, 0x2a, 0x77, 0x63, 0x43, - 0x91, 0x5a, 0x3a, 0xef, 0x47, 0x4b, 0x67, 0x60, 0xba, 0xb6, 0xc6, 0xeb, 0x1a, 0x5d, 0x3f, 0x9f, - 0x85, 0x42, 0x8f, 0x74, 0x79, 0x18, 0x95, 0xf8, 0x39, 0x6b, 0x8f, 0x74, 0x59, 0x14, 0xad, 0xd2, - 0x5d, 0x84, 0x69, 0xd9, 0x2c, 0xa9, 0x17, 0x34, 0xfe, 0x42, 0x27, 0x1f, 0x7b, 0xd0, 0x89, 0xdd, - 0xc1, 0xd5, 0x32, 0xfb, 0x54, 0x64, 0x94, 0x3d, 0xbb, 0xc3, 0xca, 0x4d, 0xdf, 0x3f, 0xa9, 0x2e, - 0x31, 0x3a, 0x7d, 0xa4, 0x1b, 0x44, 0x0e, 0x7a, 0x2c, 0x67, 0x6d, 0x10, 0xd3, 0xd2, 0xb6, 0xc4, - 0x3c, 0x1e, 0xc2, 0xc2, 0x4b, 0x9e, 0x08, 0xaa, 0x15, 0x26, 0x7f, 0x65, 0x7c, 0x7a, 0x11, 0x1a, - 0xa4, 0xe0, 0xb7, 0x59, 0xfa, 0xff, 0xbd, 0x02, 0x67, 0xb6, 0xd9, 0xfe, 0x29, 0x94, 0xc7, 0xa6, - 0xc1, 0x06, 0x6f, 0x07, 0xb0, 0x6d, 0x26, 0x90, 0x17, 0xef, 0xb7, 0x44, 0x6d, 0x1b, 0xb0, 0x24, - 0x95, 0x0b, 0x15, 0xf9, 0x89, 0x91, 0xdf, 0xb2, 0x17, 0x7e, 0x55, 0x3f, 0x82, 0xb5, 0x44, 0x2f, - 0xc4, 0x16, 0x66, 0x03, 0x16, 0x87, 0xf9, 0x2a, 0xe8, 0x44, 0x29, 0xa0, 0x35, 0x4c, 0xf5, 0x0e, - 0x9c, 0x6e, 0xfb, 0x86, 0xeb, 0x27, 0x5c, 0x30, 0x81, 0x2c, 0xc3, 0x74, 0xa3, 0xb2, 0x02, 0x76, - 0x6d, 0xc3, 0x6a, 0xdb, 0x27, 0xce, 0x6b, 0x28, 0xa5, 0x59, 0x87, 0xf6, 0x9f, 0x0c, 0xe4, 0xfa, - 0x20, 0x5f, 0xd5, 0x35, 0x8e, 0x40, 0x27, 0x5b, 0xbb, 0x0b, 0x67, 0x38, 0x00, 0xfc, 0x3a, 0x9d, - 0x38, 0x2b, 0xe1, 0xe7, 0xa4, 0xde, 0x67, 0x70, 0x6a, 0xb8, 0x2c, 0x0e, 0x31, 0x9b, 0x5b, 0x51, - 0xcc, 0xe6, 0xc2, 0x88, 0x51, 0x8f, 0x40, 0x36, 0x7f, 0x9e, 0x0b, 0xe5, 0xf5, 0x0c, 0xc4, 0xe6, - 0x6e, 0x14, 0xb1, 0xb9, 0x38, 0x4e, 0x77, 0x04, 0xb0, 0x49, 0x46, 0x6d, 0x3e, 0x25, 0x6a, 0x3f, - 0x4f, 0xc0, 0x3a, 0xb3, 0x59, 0xb8, 0x58, 0xcc, 0xda, 0xdf, 0x08, 0xaa, 0xa3, 0x71, 0x54, 0x27, - 0x68, 0x3a, 0xc0, 0xeb, 0x6f, 0xc7, 0x50, 0x9d, 0x8d, 0xb1, 0xf6, 0x06, 0xa0, 0xce, 0x5f, 0xcf, - 0x42, 0x31, 0xf8, 0x96, 0xf0, 0x79, 0xd2, 0x6d, 0xb9, 0x14, 0xb7, 0x85, 0x57, 0xe0, 0xfc, 0x37, - 0x5a, 0x81, 0x67, 0x27, 0x5e, 0x81, 0xcf, 0x41, 0x91, 0x3d, 0xe8, 0x2e, 0x3e, 0x12, 0x2b, 0x6a, - 0x81, 0x11, 0x34, 0x7c, 0x34, 0x0c, 0xc3, 0xf9, 0xa9, 0xc2, 0x30, 0x86, 0x23, 0x2d, 0xc4, 0x71, - 0xa4, 0xfb, 0xc1, 0x8a, 0xc8, 0x17, 0xd1, 0xcb, 0x23, 0xf4, 0xa6, 0xae, 0x85, 0xcd, 0xe8, 0x5a, - 0xc8, 0xd7, 0xd5, 0xf7, 0x46, 0x69, 0x19, 0xb9, 0x0a, 0x7e, 0x9b, 0x2b, 0xc4, 0x01, 0x07, 0x87, - 0xc2, 0xb1, 0x28, 0x32, 0xeb, 0x5d, 0x80, 0x20, 0x89, 0x48, 0x84, 0xe8, 0xdc, 0x88, 0x3e, 0x6a, - 0x21, 0x76, 0xaa, 0x36, 0x32, 0x34, 0xc3, 0x33, 0xa9, 0xc9, 0xf2, 0x63, 0xc6, 0x81, 0xd4, 0xff, - 0xce, 0x85, 0xf2, 0x4b, 0xc6, 0x21, 0xce, 0xfd, 0x04, 0x7e, 0x39, 0x65, 0x14, 0xdf, 0x8a, 0xc2, - 0x97, 0xaf, 0x19, 0x75, 0x09, 0xf4, 0x92, 0x55, 0x2e, 0x86, 0x2b, 0x3e, 0x73, 0xd0, 0xa8, 0x28, - 0x28, 0x75, 0xb6, 0x33, 0x38, 0xb2, 0x6c, 0xcb, 0x3b, 0xe6, 0xdf, 0xe7, 0xf9, 0xce, 0x40, 0x92, - 0xea, 0xec, 0xbe, 0x14, 0x7e, 0x65, 0xf9, 0x7a, 0x87, 0x98, 0x98, 0xc5, 0xf4, 0x9c, 0x56, 0xa0, - 0x84, 0x6d, 0x62, 0xe2, 0xe1, 0xcc, 0x2b, 0xbc, 0xde, 0xcc, 0x2b, 0xc6, 0x66, 0xde, 0x19, 0x98, - 0x77, 0xb1, 0xe1, 0x11, 0x5b, 0x6c, 0xcf, 0xc5, 0x1b, 0x1d, 0x9a, 0x3e, 0xf6, 0x3c, 0xda, 0x92, - 0x28, 0xd7, 0xc4, 0x6b, 0xa8, 0xcc, 0x5c, 0x1c, 0x5b, 0x66, 0x8e, 0x38, 0x1c, 0x8a, 0x95, 0x99, - 0xe5, 0xb1, 0x65, 0xe6, 0x44, 0x67, 0x43, 0xc3, 0x42, 0x7b, 0x69, 0xb2, 0x42, 0x3b, 0x5c, 0x97, - 0x2e, 0x47, 0xea, 0xd2, 0x6f, 0x73, 0xb2, 0xfe, 0x4a, 0x81, 0xb5, 0xc4, 0xb4, 0x12, 0xd3, 0xf5, - 0x76, 0xec, 0xf4, 0x68, 0x63, 0xac, 0xcf, 0x82, 0xc3, 0xa3, 0xc7, 0x91, 0xc3, 0xa3, 0x0f, 0xc6, - 0x0b, 0xbe, 0xf1, 0xb3, 0xa3, 0x3f, 0x52, 0xe0, 0xed, 0x03, 0xc7, 0x8c, 0x55, 0x78, 0x62, 0xdb, - 0x3f, 0x79, 0xe2, 0xb8, 0x2f, 0x6b, 0xfd, 0xdc, 0xb4, 0x38, 0x0b, 0x97, 0x53, 0x55, 0xb8, 0x90, - 0x6d, 0x86, 0x28, 0x99, 0x7e, 0x04, 0xcb, 0xbb, 0xaf, 0x70, 0xa7, 0x7d, 0x62, 0x77, 0xa6, 0x30, - 0xad, 0x02, 0xf9, 0x4e, 0xdf, 0x14, 0x28, 0x29, 0x7d, 0x0c, 0x57, 0x81, 0xf9, 0x68, 0x15, 0xa8, - 0x43, 0x65, 0xd8, 0x82, 0x18, 0xde, 0x33, 0x74, 0x78, 0x4d, 0xca, 0x4c, 0x95, 0x2f, 0x6a, 0xe2, - 0x4d, 0xd0, 0xb1, 0xcb, 0xaf, 0x44, 0x70, 0x3a, 0x76, 0xdd, 0x68, 0xb6, 0xc8, 0x47, 0xb3, 0x85, - 0xfa, 0x67, 0x0a, 0x94, 0x68, 0x0b, 0xdf, 0xc8, 0x7e, 0xb1, 0xd5, 0xca, 0x0f, 0xb7, 0x5a, 0xc1, - 0x8e, 0x6d, 0x36, 0xbc, 0x63, 0x1b, 0x5a, 0x3e, 0xc7, 0xc8, 0x49, 0xcb, 0xe7, 0x03, 0x3a, 0x76, - 0x5d, 0xf5, 0x02, 0x2c, 0x72, 0xdb, 0x44, 0xcf, 0x2b, 0x90, 0x1f, 0xb8, 0x3d, 0x19, 0x47, 0x03, - 0xb7, 0xa7, 0xfe, 0xb1, 0x02, 0xe5, 0xba, 0xef, 0x1b, 0x9d, 0xe3, 0x29, 0x3a, 0x10, 0x18, 0x97, - 0x0b, 0x1b, 0x97, 0xec, 0xc4, 0xd0, 0xdc, 0xd9, 0x0c, 0x73, 0xe7, 0x22, 0xe6, 0xaa, 0xb0, 0x24, - 0x6d, 0xc9, 0x34, 0xb8, 0x09, 0xa8, 0x45, 0x5c, 0xff, 0x11, 0x71, 0x5f, 0x1a, 0xae, 0x39, 0xdd, - 0x0e, 0x0c, 0xc1, 0xac, 0xb8, 0x43, 0x9b, 0xbf, 0x32, 0xa7, 0xb1, 0x67, 0xf5, 0x32, 0x9c, 0x8a, - 0xe8, 0xcb, 0x6c, 0xf8, 0x01, 0x94, 0x58, 0xde, 0x17, 0xa5, 0xf8, 0xcd, 0xf0, 0x71, 0xcd, 0x44, - 0xab, 0x84, 0xfa, 0xbb, 0xb0, 0x42, 0xeb, 0x03, 0x46, 0x0f, 0xa6, 0xe2, 0xf7, 0x62, 0x75, 0xea, - 0xf9, 0x0c, 0x45, 0xb1, 0x1a, 0xf5, 0x6f, 0x14, 0x98, 0x63, 0xf4, 0xc4, 0x9a, 0x7d, 0x0e, 0x8a, - 0x2e, 0x76, 0x88, 0xee, 0x1b, 0xdd, 0xe0, 0xc6, 0x32, 0x25, 0xec, 0x1b, 0x5d, 0x8f, 0x5d, 0xb8, - 0xa6, 0x1f, 0x4d, 0xab, 0x8b, 0x3d, 0x5f, 0x5e, 0x5b, 0x2e, 0x51, 0xda, 0x0e, 0x27, 0x51, 0x27, - 0x79, 0xd6, 0xef, 0xf3, 0xba, 0x73, 0x56, 0x63, 0xcf, 0x68, 0x93, 0x5f, 0xa2, 0x9b, 0x04, 0x52, - 0x67, 0x57, 0xec, 0x6a, 0x50, 0x88, 0xa1, 0xe8, 0xc1, 0xbb, 0xba, 0x0b, 0x28, 0xec, 0x05, 0xe1, - 0xef, 0x1b, 0x30, 0xcf, 0x9c, 0x24, 0xab, 0xa3, 0xb5, 0x0c, 0x37, 0x68, 0x82, 0x4d, 0x35, 0x00, - 0x71, 0x07, 0x47, 0x2a, 0xa2, 0xe9, 0x47, 0x65, 0x44, 0x85, 0xf4, 0x77, 0x0a, 0x9c, 0x8a, 0xb4, - 0x21, 0x6c, 0xbd, 0x1e, 0x6d, 0x24, 0xd3, 0x54, 0xd1, 0xc0, 0x76, 0x64, 0x49, 0xb8, 0x91, 0x65, - 0xd2, 0xaf, 0x69, 0x39, 0xf8, 0x07, 0x05, 0xa0, 0x3e, 0xf0, 0x8f, 0x05, 0x32, 0x18, 0x1e, 0x19, - 0x25, 0x3a, 0x32, 0xf4, 0x9b, 0x63, 0x78, 0xde, 0x4b, 0xe2, 0xca, 0x3d, 0x4d, 0xf0, 0xce, 0x30, - 0xbc, 0x81, 0x7f, 0x2c, 0x8f, 0xc2, 0xe8, 0x33, 0xba, 0x08, 0x4b, 0xfc, 0x96, 0xbc, 0x6e, 0x98, - 0xa6, 0x8b, 0x3d, 0x4f, 0x9c, 0x89, 0x95, 0x39, 0xb5, 0xce, 0x89, 0x94, 0xcd, 0x32, 0xb1, 0xed, - 0x5b, 0xfe, 0x89, 0xee, 0x93, 0xe7, 0xd8, 0x16, 0x7b, 0x93, 0xb2, 0xa4, 0xee, 0x53, 0x22, 0x3f, - 0x1c, 0xe8, 0x5a, 0x9e, 0xef, 0x4a, 0x36, 0x79, 0xfe, 0x22, 0xa8, 0x8c, 0x8d, 0x0e, 0x4a, 0xa5, - 0x35, 0xe8, 0xf5, 0xb8, 0x8b, 0x5f, 0x7f, 0xd8, 0xdf, 0x17, 0x1d, 0xca, 0x65, 0xc5, 0xf4, 0xd0, - 0x69, 0xa2, 0xbb, 0x6f, 0x10, 0x84, 0x79, 0x1f, 0x56, 0x42, 0x7d, 0x10, 0x61, 0x15, 0x29, 0x22, - 0x95, 0x68, 0x11, 0xa9, 0x3e, 0x06, 0xc4, 0x71, 0x87, 0x6f, 0xd8, 0x6f, 0xf5, 0x34, 0x9c, 0x8a, - 0x28, 0x12, 0x2b, 0xf1, 0x35, 0x28, 0x8b, 0x9b, 0x4e, 0x22, 0x50, 0xce, 0x42, 0x81, 0x66, 0xd4, - 0x8e, 0x65, 0xca, 0x73, 0xd2, 0x05, 0x87, 0x98, 0xdb, 0x96, 0xe9, 0xaa, 0x9f, 0x40, 0x59, 0xe3, - 0xed, 0x08, 0xde, 0x47, 0xb0, 0x24, 0xee, 0x45, 0xe9, 0x91, 0x8b, 0x89, 0x69, 0x17, 0xdf, 0xc3, - 0x8d, 0x68, 0x65, 0x3b, 0xfc, 0xaa, 0x9a, 0x50, 0xe3, 0x25, 0x43, 0x44, 0xbd, 0xec, 0xec, 0x23, - 0x90, 0x17, 0x01, 0xc6, 0xb6, 0x12, 0x95, 0x2f, 0xbb, 0xe1, 0x57, 0xf5, 0x3c, 0x9c, 0x4b, 0x6d, - 0x45, 0x78, 0xc2, 0x81, 0xca, 0xf0, 0x83, 0x69, 0xc9, 0x03, 0x63, 0x76, 0x10, 0xac, 0x84, 0x0e, - 0x82, 0xcf, 0x04, 0x45, 0x62, 0x4e, 0x2e, 0x62, 0xac, 0x02, 0x1c, 0x96, 0xfb, 0xf9, 0xac, 0x72, - 0x7f, 0x36, 0x52, 0xee, 0xab, 0xed, 0xc0, 0x9f, 0x62, 0x1b, 0xf6, 0x90, 0x6d, 0x17, 0x79, 0xdb, - 0x32, 0x21, 0xaa, 0xa3, 0x7a, 0xc9, 0x59, 0xb5, 0x90, 0x94, 0x7a, 0x15, 0xca, 0xd1, 0xd4, 0x18, - 0xca, 0x73, 0x4a, 0x22, 0xcf, 0x2d, 0xc5, 0x52, 0xdc, 0x87, 0xb1, 0x0a, 0x38, 0xdb, 0xc7, 0xb1, - 0xfa, 0xf7, 0x5e, 0x24, 0xd9, 0x5d, 0x4b, 0x39, 0xc3, 0xfd, 0x35, 0xe5, 0xb9, 0x55, 0xb1, 0x1e, - 0x3c, 0xf2, 0xa8, 0xbc, 0xe8, 0xb4, 0xfa, 0x0e, 0x94, 0x0e, 0xb2, 0x7e, 0x55, 0x31, 0x2b, 0xef, - 0x4b, 0xdc, 0x82, 0xd5, 0x47, 0x56, 0x0f, 0x7b, 0x27, 0x9e, 0x8f, 0xfb, 0x0d, 0x96, 0x94, 0x8e, - 0x2c, 0xec, 0xa2, 0x75, 0x00, 0xb6, 0x85, 0x71, 0x88, 0x15, 0x5c, 0xb6, 0x0f, 0x51, 0xd4, 0xff, - 0x54, 0x60, 0x79, 0x28, 0x78, 0xc0, 0xb6, 0x6e, 0x6f, 0x41, 0x91, 0xf6, 0xd7, 0xf3, 0x8d, 0xbe, - 0x23, 0xcf, 0xb3, 0x02, 0x02, 0xba, 0x0b, 0x73, 0x47, 0x9e, 0x84, 0x8c, 0x52, 0x01, 0xf4, 0x34, - 0x43, 0xb4, 0xd9, 0x23, 0xaf, 0x61, 0xa2, 0x8f, 0x00, 0x06, 0x1e, 0x36, 0xc5, 0x19, 0x56, 0x3e, - 0xab, 0x5a, 0x38, 0x08, 0x9f, 0x6f, 0x53, 0x01, 0x7e, 0xd5, 0xe2, 0x1e, 0x94, 0x2c, 0x9b, 0x98, - 0x98, 0x9d, 0x39, 0x9a, 0x02, 0x55, 0x1a, 0x23, 0x0e, 0x5c, 0xe2, 0xc0, 0xc3, 0xa6, 0x8a, 0xc5, - 0x5a, 0x28, 0xfd, 0x2b, 0x02, 0xa5, 0x09, 0x2b, 0x3c, 0x69, 0x1d, 0x05, 0x86, 0xcb, 0x88, 0xdd, - 0x18, 0xd5, 0x3b, 0xe6, 0x2d, 0xad, 0x62, 0x89, 0xd2, 0x46, 0x8a, 0xaa, 0x77, 0xe0, 0x74, 0x64, - 0x87, 0x34, 0xc5, 0x96, 0x45, 0x6d, 0xc5, 0x80, 0x92, 0x61, 0x38, 0x0b, 0x18, 0x42, 0x46, 0xf3, - 0x38, 0x18, 0xc2, 0xe3, 0x30, 0x84, 0xa7, 0x7e, 0x0e, 0x67, 0x23, 0x88, 0x4e, 0xc4, 0xa2, 0x7b, - 0xb1, 0xca, 0xed, 0xd2, 0x38, 0xad, 0xb1, 0x12, 0xee, 0x7f, 0x14, 0x58, 0x4d, 0x63, 0x78, 0x4d, - 0xc4, 0xf1, 0x47, 0x19, 0xf7, 0xef, 0x6e, 0x4f, 0x66, 0xd6, 0x6f, 0x04, 0xad, 0xdd, 0x87, 0x5a, - 0x9a, 0x3f, 0x93, 0xa3, 0x94, 0x9f, 0x66, 0x94, 0x7e, 0x96, 0x0f, 0x21, 0xef, 0x75, 0xdf, 0x77, - 0xad, 0xc3, 0x01, 0x0d, 0xf9, 0x37, 0x8e, 0x66, 0x35, 0x02, 0x5c, 0x86, 0xbb, 0xf6, 0xe6, 0x08, - 0xf1, 0xa1, 0x1d, 0xa9, 0xd8, 0xcc, 0xa7, 0x51, 0x6c, 0x86, 0x63, 0xea, 0xb7, 0x26, 0xd3, 0xf7, - 0x5b, 0x0b, 0x80, 0xfe, 0x2c, 0x07, 0x4b, 0xd1, 0x21, 0x42, 0xbb, 0x00, 0x46, 0x60, 0xb9, 0x98, - 0x28, 0x17, 0x27, 0xea, 0xa6, 0x16, 0x12, 0x44, 0xef, 0x41, 0xbe, 0xe3, 0x0c, 0xc4, 0xa8, 0xa5, - 0x1c, 0x06, 0x6f, 0x3b, 0x03, 0x9e, 0x51, 0x28, 0x1b, 0xdd, 0x53, 0xf1, 0xb3, 0xfd, 0xec, 0x2c, - 0xf9, 0x8c, 0x7d, 0xe7, 0x32, 0x82, 0x19, 0x3d, 0x81, 0xa5, 0x97, 0xae, 0xe5, 0x1b, 0x87, 0x3d, - 0xac, 0xf7, 0x8c, 0x13, 0xec, 0x8a, 0x2c, 0x39, 0x41, 0x22, 0x2b, 0x4b, 0xc1, 0xa7, 0x54, 0x4e, - 0xfd, 0x43, 0x28, 0x48, 0x8b, 0xc6, 0xac, 0x08, 0xfb, 0xb0, 0x36, 0xa0, 0x6c, 0x3a, 0xbb, 0x02, - 0x67, 0x1b, 0x36, 0xd1, 0x3d, 0x4c, 0x97, 0x71, 0x79, 0xdd, 0x7f, 0x4c, 0x8a, 0x5e, 0x65, 0xd2, - 0xdb, 0xc4, 0xc5, 0x4d, 0xc3, 0x26, 0x6d, 0x2e, 0xaa, 0xbe, 0x80, 0x52, 0xa8, 0x83, 0x63, 0x4c, - 0x68, 0xc0, 0x8a, 0x3c, 0x8a, 0xf7, 0xb0, 0x2f, 0x96, 0x97, 0x89, 0x1a, 0x5f, 0x16, 0x72, 0x6d, - 0xec, 0xf3, 0xeb, 0x13, 0xf7, 0xe0, 0xac, 0x86, 0x89, 0x83, 0xed, 0x60, 0x3c, 0x9f, 0x92, 0xee, - 0x14, 0x19, 0xfc, 0x2d, 0xa8, 0xa5, 0xc9, 0xf3, 0xfc, 0x70, 0xed, 0x12, 0x14, 0xe4, 0x4f, 0x64, - 0xd1, 0x02, 0xe4, 0xf7, 0xb7, 0x5b, 0x95, 0x19, 0xfa, 0x70, 0xb0, 0xd3, 0xaa, 0x28, 0xa8, 0x00, - 0xb3, 0xed, 0xed, 0xfd, 0x56, 0x25, 0x77, 0xad, 0x0f, 0x95, 0xf8, 0xef, 0x43, 0xd1, 0x1a, 0x9c, - 0x6a, 0x69, 0x7b, 0xad, 0xfa, 0xe3, 0xfa, 0x7e, 0x63, 0xaf, 0xa9, 0xb7, 0xb4, 0xc6, 0xc7, 0xf5, - 0xfd, 0xdd, 0xca, 0x0c, 0xda, 0x80, 0xf3, 0xe1, 0x0f, 0x4f, 0xf6, 0xda, 0xfb, 0xfa, 0xfe, 0x9e, - 0xbe, 0xbd, 0xd7, 0xdc, 0xaf, 0x37, 0x9a, 0xbb, 0x5a, 0x45, 0x41, 0xe7, 0xe1, 0x6c, 0x98, 0xe5, - 0x61, 0x63, 0xa7, 0xa1, 0xed, 0x6e, 0xd3, 0xe7, 0xfa, 0xd3, 0x4a, 0xee, 0xda, 0x4d, 0x28, 0x47, - 0x7e, 0xce, 0x49, 0x4d, 0x6a, 0xed, 0xed, 0x54, 0x66, 0x50, 0x19, 0x8a, 0x61, 0x3d, 0x05, 0x98, - 0x6d, 0xee, 0xed, 0xec, 0x56, 0x72, 0xd7, 0xee, 0xc0, 0x72, 0xec, 0xda, 0x2e, 0x5a, 0x81, 0x72, - 0xbb, 0xde, 0xdc, 0x79, 0xb8, 0xf7, 0xa9, 0xae, 0xed, 0xd6, 0x77, 0x3e, 0xab, 0xcc, 0xa0, 0x55, - 0xa8, 0x48, 0x52, 0x73, 0x6f, 0x9f, 0x53, 0x95, 0x6b, 0xcf, 0x63, 0x73, 0x0c, 0xa3, 0xd3, 0xb0, - 0x12, 0x34, 0xa3, 0x6f, 0x6b, 0xbb, 0xf5, 0xfd, 0x5d, 0xda, 0x7a, 0x84, 0xac, 0x1d, 0x34, 0x9b, - 0x8d, 0xe6, 0xe3, 0x8a, 0x42, 0xb5, 0x0e, 0xc9, 0xbb, 0x9f, 0x36, 0x28, 0x73, 0x2e, 0xca, 0x7c, - 0xd0, 0xfc, 0x41, 0x73, 0xef, 0x93, 0x66, 0x25, 0xbf, 0xf5, 0x8b, 0x15, 0x58, 0x92, 0x85, 0x1e, - 0x76, 0xd9, 0xad, 0x96, 0x16, 0x2c, 0xc8, 0x9f, 0x5c, 0xa7, 0x64, 0xe8, 0xe8, 0x0f, 0xc5, 0x6b, - 0x1b, 0x23, 0x38, 0x44, 0xbd, 0x3d, 0x83, 0x0e, 0x59, 0xfd, 0x1b, 0xba, 0x46, 0x7d, 0x29, 0xb5, - 0xda, 0x4c, 0xdc, 0xdc, 0xae, 0x5d, 0x1e, 0xcb, 0x17, 0xb4, 0x81, 0x69, 0x89, 0x1b, 0xfe, 0x41, - 0x11, 0xba, 0x9c, 0x56, 0x9b, 0xa6, 0xfc, 0x62, 0xa9, 0x76, 0x65, 0x3c, 0x63, 0xd0, 0xcc, 0x73, - 0xa8, 0xc4, 0x7f, 0x5c, 0x84, 0x52, 0xa0, 0xd3, 0x8c, 0x5f, 0x30, 0xd5, 0xae, 0x4d, 0xc2, 0x1a, - 0x6e, 0x2c, 0xf1, 0x33, 0x9c, 0xab, 0x93, 0xfc, 0x5c, 0x21, 0xb3, 0xb1, 0xac, 0x5f, 0x36, 0x70, - 0x07, 0x46, 0x6f, 0x3e, 0xa3, 0xd4, 0xdf, 0xbc, 0xa4, 0x5c, 0xb0, 0x4f, 0x73, 0x60, 0xfa, 0x25, - 0x6a, 0x75, 0x06, 0x1d, 0xc3, 0x72, 0xec, 0x7a, 0x02, 0x4a, 0x11, 0x4f, 0xbf, 0x87, 0x51, 0xbb, - 0x3a, 0x01, 0x67, 0x34, 0x22, 0xc2, 0xd7, 0x11, 0xd2, 0x23, 0x22, 0xe5, 0xb2, 0x43, 0x7a, 0x44, - 0xa4, 0xde, 0x6c, 0x60, 0xc1, 0x1d, 0xb9, 0x86, 0x90, 0x16, 0xdc, 0x69, 0x97, 0x1f, 0x6a, 0x97, - 0xc7, 0xf2, 0x85, 0x9d, 0x16, 0xbb, 0x94, 0x90, 0xe6, 0xb4, 0xf4, 0x4b, 0x0f, 0xb5, 0xab, 0x13, - 0x70, 0xc6, 0xa3, 0x60, 0x78, 0xc4, 0x99, 0x15, 0x05, 0x89, 0x03, 0xf9, 0xac, 0x28, 0x48, 0x9e, - 0x96, 0x8a, 0x28, 0x88, 0x1d, 0x4d, 0x5e, 0x99, 0xe0, 0x28, 0x25, 0x3b, 0x0a, 0xd2, 0x0f, 0x5d, - 0xd4, 0x19, 0xf4, 0x53, 0x05, 0xaa, 0x59, 0xc7, 0x14, 0x28, 0xa5, 0xbe, 0x1b, 0x73, 0xb2, 0x52, - 0xdb, 0x9a, 0x46, 0x24, 0xb0, 0xe2, 0x4b, 0x40, 0xc9, 0x75, 0x0f, 0x7d, 0x27, 0x6d, 0x64, 0x32, - 0x56, 0xd7, 0xda, 0x7b, 0x93, 0x31, 0x07, 0x4d, 0xb6, 0xa1, 0x20, 0x0f, 0x46, 0x50, 0x4a, 0x96, - 0x8e, 0x1d, 0xcb, 0xd4, 0xd4, 0x51, 0x2c, 0x81, 0xd2, 0xc7, 0x30, 0x4b, 0xa9, 0xe8, 0x7c, 0x3a, - 0xb7, 0x54, 0xb6, 0x9e, 0xf5, 0x39, 0x50, 0xf4, 0x0c, 0xe6, 0xf9, 0x49, 0x00, 0x4a, 0x41, 0x1e, - 0x22, 0xe7, 0x15, 0xb5, 0x0b, 0xd9, 0x0c, 0x81, 0xba, 0x2f, 0xf8, 0x7f, 0xe3, 0x10, 0x20, 0x3f, - 0x7a, 0x37, 0xfd, 0xe7, 0xcd, 0xd1, 0x33, 0x85, 0xda, 0xc5, 0x31, 0x5c, 0xe1, 0x49, 0x11, 0xab, - 0x7a, 0x2f, 0x8f, 0xdd, 0xba, 0x64, 0x4f, 0x8a, 0xf4, 0xcd, 0x11, 0x0f, 0x92, 0xe4, 0xe6, 0x29, - 0x2d, 0x48, 0x32, 0xb7, 0xac, 0x69, 0x41, 0x92, 0xbd, 0x1f, 0x53, 0x67, 0x90, 0x0f, 0xa7, 0x52, - 0xa0, 0x32, 0xf4, 0x5e, 0x56, 0x90, 0xa7, 0xe1, 0x76, 0xb5, 0xeb, 0x13, 0x72, 0x87, 0x07, 0x5f, - 0x4c, 0xfa, 0xb7, 0xb3, 0xf1, 0xa3, 0xcc, 0xc1, 0x8f, 0x4f, 0xf1, 0xad, 0x7f, 0xc9, 0xc3, 0x22, - 0x87, 0x41, 0x45, 0x05, 0xf3, 0x19, 0xc0, 0xf0, 0x04, 0x02, 0xbd, 0x93, 0xee, 0x93, 0xc8, 0x29, - 0x4d, 0xed, 0xdd, 0xd1, 0x4c, 0xe1, 0x40, 0x0b, 0xa1, 0xf9, 0x69, 0x81, 0x96, 0x3c, 0xb4, 0x48, - 0x0b, 0xb4, 0x94, 0x23, 0x01, 0x75, 0x06, 0x7d, 0x0c, 0xc5, 0x00, 0x36, 0x46, 0x69, 0xb0, 0x73, - 0x0c, 0x17, 0xaf, 0xbd, 0x33, 0x92, 0x27, 0x6c, 0x75, 0x08, 0x13, 0x4e, 0xb3, 0x3a, 0x89, 0x3d, - 0xa7, 0x59, 0x9d, 0x06, 0x2c, 0x0f, 0x7d, 0xc2, 0x91, 0xa3, 0x4c, 0x9f, 0x44, 0x80, 0xbb, 0x4c, - 0x9f, 0x44, 0xe1, 0x27, 0x75, 0xe6, 0xe1, 0xa5, 0x5f, 0x7e, 0xb5, 0xae, 0xfc, 0xd3, 0x57, 0xeb, - 0x33, 0x3f, 0xf9, 0x7a, 0x5d, 0xf9, 0xe5, 0xd7, 0xeb, 0xca, 0x3f, 0x7e, 0xbd, 0xae, 0xfc, 0xeb, - 0xd7, 0xeb, 0xca, 0x9f, 0xfe, 0xdb, 0xfa, 0xcc, 0x0f, 0x0b, 0x52, 0xfa, 0x70, 0x9e, 0xfd, 0x4f, - 0x9d, 0x0f, 0xfe, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x11, 0x3f, 0x6e, 0x39, 0x19, 0x49, 0x00, 0x00, + 0x66, 0x92, 0x5c, 0x02, 0x24, 0xc8, 0x1f, 0x90, 0x43, 0x50, 0x5f, 0xfd, 0xdd, 0xfc, 0xf0, 0x78, + 0x77, 0x36, 0x27, 0x75, 0xbf, 0x7e, 0xef, 0xd5, 0xab, 0x57, 0xaf, 0x5e, 0xbd, 0xfa, 0x55, 0x51, + 0x50, 0xd4, 0x6c, 0x63, 0xcb, 0x76, 0x2c, 0xcf, 0x42, 0x55, 0x67, 0x64, 0x7a, 0xc6, 0x10, 0x6f, + 0xbd, 0xb8, 0xa9, 0x0d, 0xec, 0x63, 0x6d, 0xbb, 0x7e, 0xbd, 0x6f, 0x78, 0xc7, 0xa3, 0xc3, 0xad, + 0x9e, 0x35, 0xbc, 0xd1, 0xb7, 0xfa, 0xd6, 0x0d, 0xca, 0x78, 0x38, 0x3a, 0xa2, 0x6f, 0xf4, 0x85, + 0x3e, 0x31, 0x05, 0xf2, 0x35, 0xa8, 0x7c, 0x8c, 0x1d, 0xd7, 0xb0, 0x4c, 0x05, 0x7f, 0x39, 0xc2, + 0xae, 0x87, 0x6a, 0xb0, 0xf4, 0x82, 0x51, 0x6a, 0xd2, 0x05, 0xe9, 0x4a, 0x51, 0x11, 0xaf, 0xf2, + 0x5f, 0x48, 0xb0, 0xe2, 0x33, 0xbb, 0xb6, 0x65, 0xba, 0x38, 0x9b, 0x1b, 0x6d, 0xc2, 0x32, 0x37, + 0x4e, 0x35, 0xb5, 0x21, 0xae, 0xe5, 0xe8, 0xe7, 0x12, 0xa7, 0xb5, 0xb5, 0x21, 0x46, 0x97, 0x61, + 0x45, 0xb0, 0x08, 0x25, 0x79, 0xca, 0x55, 0xe1, 0x64, 0xde, 0x1a, 0xda, 0x82, 0x53, 0x82, 0x51, + 0xb3, 0x0d, 0x9f, 0x79, 0x9e, 0x32, 0xaf, 0xf2, 0x4f, 0x0d, 0xdb, 0xe0, 0xfc, 0xf2, 0xe7, 0x50, + 0xdc, 0x69, 0x77, 0x9b, 0x96, 0x79, 0x64, 0xf4, 0x89, 0x89, 0x2e, 0x76, 0x88, 0x4c, 0x4d, 0xba, + 0x90, 0x27, 0x26, 0xf2, 0x57, 0x54, 0x87, 0x82, 0x8b, 0x35, 0xa7, 0x77, 0x8c, 0xdd, 0x5a, 0x8e, + 0x7e, 0xf2, 0xdf, 0x89, 0x94, 0x65, 0x7b, 0x86, 0x65, 0xba, 0xb5, 0x3c, 0x93, 0xe2, 0xaf, 0xf2, + 0xcf, 0x25, 0x28, 0x75, 0x2c, 0xc7, 0x7b, 0xa6, 0xd9, 0xb6, 0x61, 0xf6, 0xd1, 0x2d, 0x28, 0x50, + 0x5f, 0xf6, 0xac, 0x01, 0xf5, 0x41, 0x65, 0xbb, 0xbe, 0x15, 0x1f, 0x96, 0xad, 0x0e, 0xe7, 0x50, + 0x7c, 0x5e, 0x74, 0x11, 0x2a, 0x3d, 0xcb, 0xf4, 0x34, 0xc3, 0xc4, 0x8e, 0x6a, 0x5b, 0x8e, 0x47, + 0x5d, 0xb4, 0xa0, 0x94, 0x7d, 0x2a, 0x69, 0x05, 0x9d, 0x83, 0xe2, 0xb1, 0xe5, 0x7a, 0x8c, 0x23, + 0x4f, 0x39, 0x0a, 0x84, 0x40, 0x3f, 0xae, 0xc3, 0x12, 0xfd, 0x68, 0xd8, 0xdc, 0x19, 0x8b, 0xe4, + 0xb5, 0x65, 0xcb, 0xbf, 0x92, 0x60, 0xe1, 0x99, 0x35, 0x32, 0xbd, 0x58, 0x33, 0x9a, 0x77, 0xcc, + 0x07, 0x2a, 0xd4, 0x8c, 0xe6, 0x1d, 0x07, 0xcd, 0x10, 0x0e, 0x36, 0x56, 0xac, 0x19, 0xf2, 0xb1, + 0x0e, 0x05, 0x07, 0x6b, 0xba, 0x65, 0x0e, 0x4e, 0xa8, 0x09, 0x05, 0xc5, 0x7f, 0x27, 0x83, 0xe8, + 0xe2, 0x81, 0x61, 0x8e, 0x5e, 0xa9, 0x0e, 0x1e, 0x68, 0x87, 0x78, 0x40, 0x4d, 0x29, 0x28, 0x15, + 0x4e, 0x56, 0x18, 0x15, 0xed, 0x40, 0xc9, 0x76, 0x2c, 0x5b, 0xeb, 0x6b, 0xc4, 0x8f, 0xb5, 0x05, + 0xea, 0x2a, 0x39, 0xe9, 0x2a, 0x6a, 0x76, 0x27, 0xe0, 0x54, 0xc2, 0x62, 0xf2, 0x5f, 0x49, 0xb0, + 0x42, 0x82, 0xc7, 0xb5, 0xb5, 0x1e, 0xde, 0xa3, 0x43, 0x82, 0x6e, 0xc3, 0x92, 0x89, 0xbd, 0x97, + 0x96, 0xf3, 0x9c, 0x0f, 0xc0, 0xdb, 0x49, 0xad, 0xbe, 0xcc, 0x33, 0x4b, 0xc7, 0x8a, 0xe0, 0x47, + 0x37, 0x21, 0x6f, 0x1b, 0x3a, 0xed, 0xf0, 0x14, 0x62, 0x84, 0x97, 0x88, 0x18, 0x76, 0x8f, 0xfa, + 0x61, 0x1a, 0x11, 0xc3, 0xee, 0xc9, 0x32, 0x40, 0xcb, 0xf4, 0x6e, 0x7d, 0xf7, 0x63, 0x6d, 0x30, + 0xc2, 0x68, 0x0d, 0x16, 0x5e, 0x90, 0x07, 0x6a, 0x6c, 0x5e, 0x61, 0x2f, 0xf2, 0x57, 0x79, 0x38, + 0xf7, 0x94, 0xf8, 0xab, 0xab, 0x99, 0xfa, 0xa1, 0xf5, 0xaa, 0x8b, 0x7b, 0x23, 0xc7, 0xf0, 0x4e, + 0x9a, 0x96, 0xe9, 0xe1, 0x57, 0x1e, 0x6a, 0xc3, 0xaa, 0x29, 0x34, 0xab, 0x22, 0x34, 0x89, 0x86, + 0xd2, 0xf6, 0xe6, 0x18, 0x23, 0x98, 0x8b, 0x94, 0xaa, 0x19, 0x25, 0xb8, 0xe8, 0x49, 0x30, 0x6e, + 0x42, 0x5b, 0x8e, 0x6a, 0x4b, 0xe9, 0x52, 0x77, 0x97, 0x5a, 0xc6, 0x75, 0x89, 0x81, 0x15, 0x9a, + 0x3e, 0x02, 0x32, 0xab, 0x55, 0xcd, 0x55, 0x47, 0x2e, 0x76, 0xa8, 0x63, 0x4a, 0xdb, 0x6f, 0x25, + 0xb5, 0x04, 0x2e, 0x50, 0x8a, 0xce, 0xc8, 0x6c, 0xb8, 0x07, 0x2e, 0x76, 0x68, 0x12, 0xe0, 0xb1, + 0xa4, 0x3a, 0x96, 0xe5, 0x1d, 0xb9, 0x22, 0x7e, 0x04, 0x59, 0xa1, 0x54, 0x74, 0x03, 0x4e, 0xb9, + 0x23, 0xdb, 0x1e, 0xe0, 0x21, 0x36, 0x3d, 0x6d, 0xa0, 0xf6, 0x1d, 0x6b, 0x64, 0xbb, 0xb5, 0x85, + 0x0b, 0xf9, 0x2b, 0x79, 0x05, 0x85, 0x3f, 0x3d, 0xa6, 0x5f, 0xd0, 0x06, 0x80, 0xed, 0x18, 0x2f, + 0x8c, 0x01, 0xee, 0x63, 0xbd, 0xb6, 0x48, 0x95, 0x86, 0x28, 0xe8, 0x7d, 0x58, 0x73, 0x71, 0xaf, + 0x67, 0x0d, 0x6d, 0xd5, 0x76, 0xac, 0x23, 0x63, 0x80, 0x59, 0xf4, 0x2f, 0xd1, 0xe8, 0x47, 0xfc, + 0x5b, 0x87, 0x7d, 0xa2, 0xf3, 0xe0, 0x1e, 0xcd, 0x69, 0xa4, 0xa7, 0xb4, 0xf1, 0x5a, 0x61, 0x8a, + 0xae, 0x02, 0xed, 0x2a, 0x35, 0x49, 0xfe, 0x79, 0x0e, 0x4e, 0x53, 0x4f, 0x76, 0x2c, 0x9d, 0x0f, + 0x33, 0x4f, 0x52, 0xef, 0x40, 0xb9, 0x47, 0x75, 0xaa, 0xb6, 0xe6, 0x60, 0xd3, 0xe3, 0x93, 0x74, + 0x99, 0x11, 0x3b, 0x94, 0x86, 0x3e, 0x85, 0xaa, 0xcb, 0xa3, 0x42, 0xed, 0xb1, 0xb0, 0xe0, 0x63, + 0x76, 0x3d, 0x69, 0xc2, 0x98, 0x58, 0x52, 0x56, 0xdc, 0x44, 0x70, 0x2d, 0xb9, 0x27, 0x6e, 0xcf, + 0x1b, 0xb0, 0x6c, 0x57, 0xda, 0xfe, 0x6e, 0x86, 0xc2, 0xb8, 0xe1, 0x5b, 0x5d, 0x26, 0xb6, 0x6b, + 0x7a, 0xce, 0x89, 0x22, 0x94, 0xd4, 0xef, 0xc0, 0x72, 0xf8, 0x03, 0xaa, 0x42, 0xfe, 0x39, 0x3e, + 0xe1, 0x9d, 0x22, 0x8f, 0xc1, 0x24, 0x60, 0xb9, 0x86, 0xbd, 0xdc, 0xc9, 0xfd, 0x8e, 0x24, 0x3b, + 0x80, 0x82, 0x56, 0x9e, 0x61, 0x4f, 0xd3, 0x35, 0x4f, 0x43, 0x08, 0xe6, 0xe9, 0x32, 0xc2, 0x54, + 0xd0, 0x67, 0xa2, 0x75, 0xc4, 0x27, 0x6f, 0x51, 0x21, 0x8f, 0xe8, 0x2d, 0x28, 0xfa, 0x81, 0xce, + 0xd7, 0x92, 0x80, 0x40, 0x72, 0xba, 0xe6, 0x79, 0x78, 0x68, 0x7b, 0x34, 0xc4, 0xca, 0x8a, 0x78, + 0x95, 0xff, 0x7b, 0x1e, 0xaa, 0x89, 0x31, 0x79, 0x00, 0x85, 0x21, 0x6f, 0x9e, 0x4f, 0xb4, 0x77, + 0x53, 0x12, 0x7b, 0xc2, 0x54, 0xc5, 0x97, 0x22, 0x79, 0x93, 0xe4, 0xd0, 0xd0, 0xfa, 0xe7, 0xbf, + 0x93, 0x11, 0x1f, 0x58, 0x7d, 0x55, 0x37, 0x1c, 0xdc, 0xf3, 0x2c, 0xe7, 0x84, 0x9b, 0xbb, 0x3c, + 0xb0, 0xfa, 0x3b, 0x82, 0x86, 0xee, 0x00, 0xe8, 0xa6, 0x4b, 0x06, 0xfb, 0xc8, 0xe8, 0x53, 0xa3, + 0x4b, 0xdb, 0xe7, 0x92, 0x46, 0xf8, 0x8b, 0x9d, 0x52, 0xd4, 0x4d, 0x97, 0x9b, 0xff, 0x10, 0xca, + 0x64, 0xcd, 0x50, 0x87, 0x6c, 0x9d, 0x62, 0x33, 0xa5, 0xb4, 0x7d, 0x3e, 0xad, 0x0f, 0xfe, 0x6a, + 0xa6, 0x2c, 0xdb, 0xc1, 0x8b, 0x8b, 0x1e, 0xc1, 0x22, 0x4d, 0xde, 0x6e, 0x6d, 0x91, 0x0a, 0x6f, + 0x8d, 0x73, 0x00, 0x8f, 0x88, 0xa7, 0x54, 0x80, 0x05, 0x04, 0x97, 0x46, 0x07, 0x50, 0xd2, 0x4c, + 0xd3, 0xf2, 0x34, 0x96, 0x68, 0x96, 0xa8, 0xb2, 0x0f, 0xa6, 0x50, 0xd6, 0x08, 0xa4, 0x98, 0xc6, + 0xb0, 0x1e, 0xf4, 0x7d, 0x58, 0xa0, 0x99, 0x88, 0x4f, 0xc4, 0xcb, 0x53, 0x06, 0xad, 0xc2, 0xa4, + 0xea, 0xb7, 0xa1, 0x14, 0x32, 0x76, 0x96, 0x20, 0xad, 0xdf, 0x83, 0x6a, 0xdc, 0xb4, 0x99, 0x82, + 0xfc, 0x0f, 0x60, 0x4d, 0x19, 0x99, 0x81, 0x61, 0xa2, 0xfa, 0xba, 0x03, 0x8b, 0x7c, 0xb0, 0x59, + 0xc4, 0xc9, 0x93, 0x7d, 0xa4, 0x70, 0x89, 0x70, 0x39, 0x75, 0xac, 0x99, 0xfa, 0x00, 0x3b, 0xbc, + 0x5d, 0x51, 0x4e, 0x3d, 0x61, 0x54, 0xf9, 0xfb, 0x70, 0x3a, 0xd6, 0x38, 0xaf, 0xe6, 0xde, 0x85, + 0x8a, 0x6d, 0xe9, 0xaa, 0xcb, 0xc8, 0xaa, 0xa1, 0x8b, 0x34, 0x64, 0xfb, 0xbc, 0x2d, 0x9d, 0x88, + 0x77, 0x3d, 0xcb, 0x4e, 0x1a, 0x3f, 0x9d, 0x78, 0x0d, 0xce, 0xc4, 0xc5, 0x59, 0xf3, 0xf2, 0x7d, + 0x58, 0x57, 0xf0, 0xd0, 0x7a, 0x81, 0x5f, 0x57, 0x75, 0x1d, 0x6a, 0x49, 0x05, 0x5c, 0xf9, 0x67, + 0xb0, 0x1e, 0x50, 0xbb, 0x9e, 0xe6, 0x8d, 0xdc, 0x99, 0x94, 0xf3, 0x52, 0xf7, 0xd0, 0x72, 0xd9, + 0x70, 0x16, 0x14, 0xf1, 0x2a, 0x5f, 0x0d, 0xab, 0x6e, 0xb3, 0xca, 0x82, 0xb5, 0x80, 0x2a, 0x90, + 0x33, 0x6c, 0xae, 0x2e, 0x67, 0xd8, 0xf2, 0x13, 0x28, 0xfa, 0x4b, 0x33, 0xba, 0x1b, 0xd4, 0x98, + 0xb9, 0x69, 0x17, 0x72, 0xbf, 0x0c, 0xdd, 0x4f, 0x2c, 0x25, 0xbc, 0xc9, 0xbb, 0x00, 0x7e, 0xca, + 0x13, 0x15, 0xc2, 0xb9, 0x31, 0x8a, 0x95, 0x10, 0xbb, 0xfc, 0xd3, 0x85, 0x70, 0x22, 0x0c, 0x75, + 0x42, 0xf7, 0x3b, 0xa1, 0x47, 0x12, 0x63, 0xee, 0xb5, 0x12, 0xe3, 0x87, 0xb0, 0xe0, 0x7a, 0x9a, + 0x87, 0x79, 0x15, 0xb5, 0x39, 0x4e, 0x9c, 0x18, 0x81, 0x15, 0xc6, 0x8f, 0xce, 0x03, 0xf4, 0x1c, + 0xac, 0x79, 0x58, 0x57, 0x35, 0x96, 0xc5, 0xf3, 0x4a, 0x91, 0x53, 0x1a, 0x1e, 0x6a, 0x06, 0x95, + 0xe0, 0x02, 0x35, 0xec, 0xea, 0x38, 0xcd, 0x91, 0xa1, 0x0a, 0x6a, 0x42, 0x3f, 0xab, 0x2c, 0x4e, + 0x99, 0x55, 0xb8, 0x02, 0x26, 0x15, 0xca, 0x99, 0x4b, 0x93, 0x73, 0x26, 0x13, 0x9d, 0x26, 0x67, + 0x16, 0x26, 0xe7, 0x4c, 0xae, 0x6c, 0x7c, 0xce, 0x4c, 0xc9, 0x12, 0xc5, 0xb4, 0x2c, 0xf1, 0x6d, + 0x66, 0xc7, 0x7f, 0x96, 0xa0, 0x96, 0x9c, 0xac, 0x3c, 0x49, 0xdd, 0x81, 0x45, 0x97, 0x52, 0xa6, + 0x49, 0x91, 0x5c, 0x96, 0x4b, 0xa0, 0x27, 0x30, 0x6f, 0x98, 0x47, 0x16, 0xdd, 0xed, 0xa5, 0x16, + 0x39, 0x59, 0xad, 0x6e, 0xb5, 0xcc, 0x23, 0x8b, 0x79, 0x93, 0x6a, 0xa8, 0x7f, 0x08, 0x45, 0x9f, + 0x34, 0x53, 0xdf, 0xf6, 0x60, 0x2d, 0x16, 0xdb, 0x6c, 0x57, 0xe0, 0x4f, 0x09, 0x69, 0xb6, 0x29, + 0x21, 0xff, 0x24, 0x17, 0x9e, 0xb2, 0x8f, 0x8c, 0x81, 0x87, 0x9d, 0xc4, 0x94, 0xfd, 0x48, 0x68, + 0x67, 0xf3, 0xf5, 0xd2, 0x44, 0xed, 0xac, 0x78, 0xe5, 0xb3, 0xee, 0x0b, 0xa8, 0xd0, 0xa0, 0x54, + 0x5d, 0x3c, 0xa0, 0x95, 0x09, 0xaf, 0x12, 0xbf, 0x37, 0x4e, 0x0d, 0xb3, 0x84, 0x85, 0x76, 0x97, + 0xcb, 0x31, 0x0f, 0x96, 0x07, 0x61, 0x5a, 0xfd, 0x01, 0xa0, 0x24, 0xd3, 0x4c, 0x3e, 0xed, 0x92, + 0x5c, 0x48, 0xb6, 0xc4, 0x29, 0xcb, 0xe9, 0x11, 0x35, 0x63, 0x9a, 0x58, 0x61, 0x06, 0x2b, 0x5c, + 0x42, 0xfe, 0xaf, 0x3c, 0x40, 0xf0, 0xf1, 0xff, 0x51, 0x12, 0x7c, 0xe0, 0x27, 0x20, 0x56, 0xf1, + 0x5d, 0x19, 0xa7, 0x38, 0x35, 0xf5, 0xec, 0x45, 0x53, 0x0f, 0xab, 0xfd, 0xae, 0x8f, 0x55, 0x33, + 0x73, 0xd2, 0x59, 0xfa, 0x6d, 0x4b, 0x3a, 0x4f, 0xe1, 0x4c, 0x3c, 0x88, 0x78, 0xc6, 0xd9, 0x86, + 0x05, 0xc3, 0xc3, 0x43, 0x86, 0x1f, 0xa5, 0xee, 0xf7, 0x42, 0x42, 0x8c, 0x55, 0xde, 0x84, 0x62, + 0x6b, 0xa8, 0xf5, 0x71, 0xd7, 0xc6, 0x3d, 0xd2, 0xa8, 0x41, 0x5e, 0xb8, 0x21, 0xec, 0x45, 0xde, + 0x86, 0xc2, 0x0f, 0xf0, 0x09, 0x9b, 0xfd, 0x53, 0x1a, 0x2a, 0xff, 0x49, 0x0e, 0xd6, 0xe9, 0xea, + 0xd3, 0x14, 0xe8, 0x8d, 0x82, 0x5d, 0x6b, 0xe4, 0xf4, 0xb0, 0x4b, 0xc3, 0xc2, 0x1e, 0xa9, 0x36, + 0x76, 0x0c, 0x4b, 0xe7, 0xe0, 0x42, 0xb1, 0x67, 0x8f, 0x3a, 0x94, 0x80, 0xce, 0x01, 0x79, 0x51, + 0xbf, 0x1c, 0x59, 0x3c, 0x62, 0xf3, 0x4a, 0xa1, 0x67, 0x8f, 0x7e, 0x8f, 0xbc, 0x0b, 0x59, 0xf7, + 0x58, 0x73, 0xb0, 0x4b, 0x03, 0x92, 0xc9, 0x76, 0x29, 0x01, 0xdd, 0x84, 0xd3, 0x43, 0x3c, 0xb4, + 0x9c, 0x13, 0x75, 0x60, 0x0c, 0x0d, 0x4f, 0x35, 0x4c, 0xf5, 0xf0, 0xc4, 0xc3, 0x2e, 0x0f, 0x3e, + 0xc4, 0x3e, 0x3e, 0x25, 0xdf, 0x5a, 0xe6, 0x43, 0xf2, 0x05, 0xc9, 0x50, 0xb6, 0xac, 0xa1, 0xea, + 0xf6, 0x2c, 0x07, 0xab, 0x9a, 0xfe, 0x63, 0xba, 0x20, 0xe7, 0x95, 0x92, 0x65, 0x0d, 0xbb, 0x84, + 0xd6, 0xd0, 0x7f, 0x8c, 0xde, 0x86, 0x52, 0xcf, 0x1e, 0xb9, 0xd8, 0x53, 0xc9, 0x1f, 0xba, 0xde, + 0x16, 0x15, 0x60, 0xa4, 0xa6, 0x3d, 0x72, 0x43, 0x0c, 0x43, 0xe2, 0xff, 0xa5, 0x30, 0xc3, 0x33, + 0xe2, 0x66, 0x0d, 0xca, 0x11, 0x70, 0x82, 0xec, 0x13, 0x29, 0x0a, 0xc1, 0xf7, 0x89, 0xe4, 0x99, + 0xd0, 0x1c, 0x6b, 0x20, 0x3c, 0x49, 0x9f, 0x09, 0xcd, 0x3b, 0xb1, 0xc5, 0x26, 0x91, 0x3e, 0x13, + 0x97, 0x0f, 0xf0, 0x0b, 0x0e, 0x60, 0x15, 0x15, 0xf6, 0x22, 0xeb, 0x00, 0x4d, 0xcd, 0xd6, 0x0e, + 0x8d, 0x81, 0xe1, 0x9d, 0xa0, 0xab, 0x50, 0xd5, 0x74, 0x5d, 0xed, 0x09, 0x8a, 0x81, 0x05, 0xac, + 0xb8, 0xa2, 0xe9, 0x7a, 0x33, 0x44, 0x46, 0xdf, 0x81, 0x55, 0xdd, 0xb1, 0xec, 0x28, 0x2f, 0xc3, + 0x19, 0xab, 0xe4, 0x43, 0x98, 0x59, 0xfe, 0xf7, 0x05, 0x38, 0x1f, 0x1d, 0xd8, 0x38, 0x00, 0xf4, + 0x00, 0x96, 0x63, 0xad, 0x66, 0x80, 0x0f, 0x81, 0xb5, 0x4a, 0x44, 0x22, 0x06, 0x88, 0xe4, 0x12, + 0x80, 0x48, 0x2a, 0xc4, 0x94, 0x7f, 0xa3, 0x10, 0xd3, 0xfc, 0x1b, 0x81, 0x98, 0x16, 0x66, 0x83, + 0x98, 0x2e, 0xd1, 0xec, 0x23, 0xa4, 0xe9, 0x6e, 0x9c, 0x85, 0x5a, 0xd9, 0xe7, 0x31, 0x05, 0x1e, + 0x1d, 0x83, 0xa2, 0x96, 0x66, 0x81, 0xa2, 0x0a, 0x99, 0x50, 0x14, 0x89, 0x1a, 0xdb, 0xd6, 0x9c, + 0xa1, 0xe5, 0x08, 0xac, 0x89, 0x57, 0x5d, 0x2b, 0x82, 0xce, 0x71, 0xa6, 0x4c, 0x54, 0x0a, 0x32, + 0x51, 0xa9, 0x0b, 0xb0, 0x6c, 0x5a, 0xaa, 0x89, 0x5f, 0xaa, 0x64, 0x2c, 0xdd, 0x5a, 0x89, 0x0d, + 0xac, 0x69, 0xb5, 0xf1, 0xcb, 0x0e, 0xa1, 0x24, 0x70, 0xab, 0xe5, 0xd9, 0x70, 0x2b, 0xb4, 0x09, + 0xcb, 0x43, 0xcd, 0x7d, 0x8e, 0x75, 0x6a, 0x8a, 0x5b, 0x2b, 0xd3, 0x20, 0x2e, 0x31, 0x1a, 0xb1, + 0xc1, 0x45, 0x17, 0xc1, 0x77, 0x12, 0x67, 0xaa, 0x50, 0xa6, 0xb2, 0xa0, 0x52, 0x36, 0xf9, 0x6f, + 0x25, 0x58, 0x8b, 0x86, 0x39, 0x47, 0x2b, 0x1e, 0x43, 0xd1, 0x11, 0x99, 0x8c, 0x87, 0xf6, 0xd5, + 0x8c, 0xc2, 0x3b, 0x99, 0xfa, 0x94, 0x40, 0x16, 0xfd, 0x30, 0x13, 0x24, 0xbb, 0x31, 0x49, 0xdf, + 0x24, 0x98, 0x4c, 0x76, 0xe0, 0xed, 0x4f, 0x0c, 0x53, 0xb7, 0x5e, 0xba, 0x99, 0xb3, 0x34, 0x25, + 0xd6, 0xa4, 0x8c, 0x58, 0xeb, 0x39, 0x58, 0xc7, 0xa6, 0x67, 0x68, 0x03, 0xd5, 0xb5, 0x71, 0x4f, + 0x6c, 0xd6, 0x03, 0x32, 0x59, 0x3b, 0xe4, 0x5f, 0x48, 0x70, 0x26, 0xde, 0x28, 0xf7, 0x59, 0x2b, + 0xe9, 0xb3, 0xef, 0x24, 0xfb, 0x18, 0x17, 0x4e, 0xf5, 0xda, 0x17, 0x99, 0x5e, 0xbb, 0x39, 0x59, + 0xe3, 0x44, 0xbf, 0xfd, 0xa5, 0x04, 0x67, 0x33, 0xcd, 0x88, 0xad, 0x3d, 0x52, 0x7c, 0xed, 0xe1, + 0xeb, 0x56, 0xcf, 0x1a, 0x99, 0x5e, 0x68, 0xdd, 0x6a, 0xd2, 0xd3, 0x0d, 0xb6, 0x40, 0xa8, 0x43, + 0xed, 0x95, 0x31, 0x1c, 0x0d, 0xf9, 0xc2, 0x45, 0xd4, 0x3d, 0x63, 0x94, 0xd7, 0x58, 0xb9, 0xe4, + 0x06, 0xac, 0xfa, 0x56, 0x8e, 0xc5, 0x1f, 0x43, 0x78, 0x62, 0x2e, 0x8a, 0x27, 0x9a, 0xb0, 0xb8, + 0x83, 0x5f, 0x18, 0x3d, 0xfc, 0x46, 0x8e, 0x5f, 0x2e, 0x40, 0xc9, 0xc6, 0xce, 0xd0, 0x70, 0x5d, + 0x3f, 0x23, 0x17, 0x95, 0x30, 0x49, 0xfe, 0x8f, 0x45, 0x58, 0x89, 0x47, 0xc7, 0xfd, 0x04, 0x7c, + 0xf9, 0x4e, 0xca, 0x5a, 0x11, 0xef, 0x68, 0xa8, 0x3e, 0xbd, 0x29, 0xaa, 0x96, 0x5c, 0x16, 0x86, + 0xe0, 0x57, 0x38, 0xbc, 0xa4, 0x21, 0x1e, 0xe9, 0x59, 0xc3, 0xa1, 0x66, 0xea, 0xe2, 0xd4, 0x8c, + 0xbf, 0x12, 0xff, 0x69, 0x4e, 0x9f, 0xb8, 0x9d, 0x90, 0xe9, 0x33, 0x19, 0x3c, 0xb2, 0xe1, 0x36, + 0x4c, 0x0a, 0x83, 0xd2, 0xac, 0x5e, 0x54, 0x80, 0x93, 0x76, 0x0c, 0x07, 0x6d, 0xc1, 0x3c, 0x36, + 0x5f, 0x88, 0x02, 0x34, 0xe5, 0x58, 0x4d, 0xd4, 0x4f, 0x0a, 0xe5, 0x43, 0x37, 0x60, 0x71, 0x48, + 0xc2, 0x42, 0x6c, 0xbd, 0xd7, 0x33, 0x4e, 0x97, 0x14, 0xce, 0x86, 0xb6, 0x61, 0x49, 0xa7, 0xe3, + 0x24, 0xf6, 0xd7, 0xb5, 0x14, 0x70, 0x95, 0x32, 0x28, 0x82, 0x11, 0xed, 0xfa, 0xe5, 0x75, 0x31, + 0xab, 0x2e, 0x8e, 0x0d, 0x45, 0x6a, 0x8d, 0xbd, 0x1f, 0xad, 0xb1, 0x81, 0xea, 0xda, 0x9e, 0xac, + 0x6b, 0x7c, 0xa1, 0x7d, 0x16, 0x0a, 0x03, 0xab, 0xcf, 0xc2, 0xa8, 0xc4, 0x0e, 0x64, 0x07, 0x56, + 0x9f, 0x46, 0xd1, 0x1a, 0xd9, 0x6e, 0xe8, 0x86, 0x49, 0xb3, 0x7f, 0x41, 0x61, 0x2f, 0x64, 0xf2, + 0xd1, 0x07, 0xd5, 0x32, 0x7b, 0xb8, 0x56, 0xa6, 0x9f, 0x8a, 0x94, 0xb2, 0x67, 0xf6, 0x68, 0x5d, + 0xea, 0x79, 0x27, 0xb5, 0x0a, 0xa5, 0x93, 0x47, 0xb2, 0x93, 0x64, 0xe8, 0xc8, 0x4a, 0xd6, 0x4e, + 0x32, 0x2d, 0xbf, 0x0b, 0x70, 0xe4, 0x21, 0x2c, 0xbd, 0x64, 0x89, 0xa0, 0x56, 0xa5, 0xf2, 0x57, + 0x26, 0xa7, 0x17, 0xae, 0x41, 0x08, 0x7e, 0x9b, 0x7b, 0x84, 0xbf, 0x97, 0xe0, 0x4c, 0x93, 0x6e, + 0xb4, 0x42, 0x79, 0x6c, 0x16, 0x10, 0xf1, 0xb6, 0x8f, 0xef, 0x66, 0x22, 0x7e, 0xf1, 0x7e, 0x0b, + 0x78, 0xb7, 0x05, 0x15, 0xa1, 0x9c, 0xab, 0xc8, 0x4f, 0x0d, 0x11, 0x97, 0xdd, 0xf0, 0xab, 0xfc, + 0x11, 0xac, 0x27, 0x7a, 0xc1, 0xf7, 0x3a, 0x9b, 0xb0, 0x1c, 0xe4, 0x2b, 0xbf, 0x13, 0x25, 0x9f, + 0xd6, 0xd2, 0xe5, 0x3b, 0x70, 0xba, 0xeb, 0x69, 0x8e, 0x97, 0x70, 0xc1, 0x14, 0xb2, 0x14, 0xfc, + 0x8d, 0xca, 0x72, 0x7c, 0xb6, 0x0b, 0x6b, 0x5d, 0xcf, 0xb2, 0x5f, 0x43, 0x29, 0xc9, 0x3a, 0xa4, + 0xff, 0xd6, 0x48, 0xac, 0x0f, 0xe2, 0x55, 0x5e, 0x67, 0x50, 0x75, 0xb2, 0xb5, 0xbb, 0x70, 0x86, + 0x21, 0xc5, 0xaf, 0xd3, 0x89, 0xb3, 0x02, 0xa7, 0x4e, 0xea, 0x7d, 0x06, 0xa7, 0x82, 0x65, 0x31, + 0x00, 0x77, 0x6e, 0x45, 0xc1, 0x9d, 0x0b, 0x63, 0x46, 0x3d, 0x82, 0xed, 0xfc, 0x79, 0x2e, 0x94, + 0xd7, 0x33, 0xa0, 0x9d, 0xbb, 0x51, 0x68, 0xe7, 0xe2, 0x24, 0xdd, 0x11, 0x64, 0x27, 0x19, 0xb5, + 0xf9, 0x94, 0xa8, 0xfd, 0x3c, 0x81, 0xff, 0xcc, 0x67, 0x01, 0x68, 0x31, 0x6b, 0x7f, 0x23, 0xf0, + 0x8f, 0xc2, 0xe0, 0x1f, 0xbf, 0x69, 0x1f, 0xd8, 0xbf, 0x1d, 0x83, 0x7f, 0x36, 0x27, 0xda, 0xeb, + 0xa3, 0x3f, 0x7f, 0x3d, 0x0f, 0x45, 0xff, 0x5b, 0xc2, 0xe7, 0x49, 0xb7, 0xe5, 0x52, 0xdc, 0x16, + 0x5e, 0x81, 0xf3, 0xdf, 0x68, 0x05, 0x9e, 0x9f, 0x7a, 0x05, 0x3e, 0x07, 0x45, 0xfa, 0xa0, 0x3a, + 0xf8, 0x88, 0xaf, 0xa8, 0x05, 0x4a, 0x50, 0xf0, 0x51, 0x10, 0x86, 0x8b, 0x33, 0x85, 0x61, 0x0c, + 0x70, 0x5a, 0x8a, 0x03, 0x4e, 0xf7, 0xfd, 0x15, 0x91, 0x2d, 0xa2, 0x97, 0xc7, 0xe8, 0x4d, 0x5d, + 0x0b, 0xdb, 0xd1, 0xb5, 0x90, 0xad, 0xab, 0xef, 0x8d, 0xd3, 0x32, 0x76, 0x15, 0xfc, 0x36, 0x57, + 0x88, 0x03, 0x86, 0x22, 0x85, 0x63, 0x91, 0x67, 0xd6, 0xbb, 0x00, 0x7e, 0x12, 0x11, 0x50, 0xd2, + 0xb9, 0x31, 0x7d, 0x54, 0x42, 0xec, 0x44, 0x6d, 0x64, 0x68, 0x82, 0xc3, 0xab, 0xe9, 0xf2, 0x63, + 0xc6, 0xc9, 0xd5, 0xff, 0x2e, 0x84, 0xf2, 0x4b, 0xc6, 0x69, 0xcf, 0xfd, 0x04, 0xd0, 0x39, 0x63, + 0x14, 0xdf, 0x8a, 0xe2, 0x9c, 0xaf, 0x19, 0x75, 0x09, 0x98, 0x93, 0x56, 0x2e, 0x9a, 0xc3, 0x3f, + 0x33, 0x74, 0xa9, 0xc8, 0x29, 0x0d, 0xba, 0x33, 0x38, 0x32, 0x4c, 0xc3, 0x3d, 0x66, 0xdf, 0x17, + 0xd9, 0xce, 0x40, 0x90, 0x1a, 0xf4, 0x62, 0x15, 0x7e, 0x65, 0x78, 0x6a, 0xcf, 0xd2, 0x31, 0x8d, + 0xe9, 0x05, 0xa5, 0x40, 0x08, 0x4d, 0x4b, 0xc7, 0xc1, 0xcc, 0x2b, 0xbc, 0xde, 0xcc, 0x2b, 0xc6, + 0x66, 0xde, 0x19, 0x58, 0x74, 0xb0, 0xe6, 0x5a, 0x26, 0xdf, 0xc7, 0xf3, 0x37, 0x32, 0x34, 0x43, + 0xec, 0xba, 0xa4, 0x25, 0x5e, 0xae, 0xf1, 0xd7, 0x50, 0x99, 0xb9, 0x3c, 0xb1, 0xcc, 0x1c, 0x73, + 0x8a, 0x14, 0x2b, 0x33, 0xcb, 0x13, 0xcb, 0xcc, 0xa9, 0x0e, 0x91, 0x82, 0x42, 0xbb, 0x32, 0x5d, + 0xa1, 0x1d, 0xae, 0x4b, 0x57, 0x22, 0x75, 0xe9, 0xb7, 0x39, 0x59, 0x7f, 0x25, 0xc1, 0x7a, 0x62, + 0x5a, 0xf1, 0xe9, 0x7a, 0x3b, 0x76, 0xcc, 0xb4, 0x39, 0xd1, 0x67, 0xfe, 0x29, 0xd3, 0xe3, 0xc8, + 0x29, 0xd3, 0x07, 0x93, 0x05, 0xdf, 0xf8, 0x21, 0xd3, 0x1f, 0x49, 0xf0, 0xf6, 0x81, 0xad, 0xc7, + 0x2a, 0x3c, 0xbe, 0xed, 0x9f, 0x3e, 0x71, 0xdc, 0x17, 0xb5, 0x7e, 0x6e, 0x56, 0x40, 0x86, 0xc9, + 0xc9, 0x32, 0x5c, 0xc8, 0x36, 0x83, 0x97, 0x4c, 0x3f, 0x82, 0x95, 0xdd, 0x57, 0xb8, 0xd7, 0x3d, + 0x31, 0x7b, 0x33, 0x98, 0x56, 0x85, 0x7c, 0x6f, 0xa8, 0x73, 0x38, 0x95, 0x3c, 0x86, 0xab, 0xc0, + 0x7c, 0xb4, 0x0a, 0x54, 0xa1, 0x1a, 0xb4, 0xc0, 0x87, 0xf7, 0x0c, 0x19, 0x5e, 0x9d, 0x30, 0x13, + 0xe5, 0xcb, 0x0a, 0x7f, 0xe3, 0x74, 0xec, 0xb0, 0xbb, 0x13, 0x8c, 0x8e, 0x1d, 0x27, 0x9a, 0x2d, + 0xf2, 0xd1, 0x6c, 0x21, 0xff, 0x99, 0x04, 0x25, 0xd2, 0xc2, 0x37, 0xb2, 0x9f, 0x6f, 0xb5, 0xf2, + 0xc1, 0x56, 0xcb, 0xdf, 0xb1, 0xcd, 0x87, 0x77, 0x6c, 0x81, 0xe5, 0x0b, 0x94, 0x9c, 0xb4, 0x7c, + 0xd1, 0xa7, 0x63, 0xc7, 0x91, 0x2f, 0xc0, 0x32, 0xb3, 0x8d, 0xf7, 0xbc, 0x0a, 0xf9, 0x91, 0x33, + 0x10, 0x71, 0x34, 0x72, 0x06, 0xf2, 0x1f, 0x4b, 0x50, 0x6e, 0x78, 0x9e, 0xd6, 0x3b, 0x9e, 0xa1, + 0x03, 0xbe, 0x71, 0xb9, 0xb0, 0x71, 0xc9, 0x4e, 0x04, 0xe6, 0xce, 0x67, 0x98, 0xbb, 0x10, 0x31, + 0x57, 0x86, 0x8a, 0xb0, 0x25, 0xd3, 0xe0, 0x36, 0xa0, 0x8e, 0xe5, 0x78, 0x8f, 0x2c, 0xe7, 0xa5, + 0xe6, 0xe8, 0xb3, 0xed, 0xc0, 0x10, 0xcc, 0xf3, 0xcb, 0xb6, 0xf9, 0x2b, 0x0b, 0x0a, 0x7d, 0x96, + 0x2f, 0xc3, 0xa9, 0x88, 0xbe, 0xcc, 0x86, 0x1f, 0x40, 0x89, 0xe6, 0x7d, 0x5e, 0x8a, 0xdf, 0x0c, + 0x9f, 0xeb, 0x4c, 0xb5, 0x4a, 0xc8, 0xbf, 0x0b, 0xab, 0xa4, 0x3e, 0xa0, 0x74, 0x7f, 0x2a, 0x7e, + 0x2f, 0x56, 0xa7, 0x9e, 0xcf, 0x50, 0x14, 0xab, 0x51, 0xff, 0x46, 0x82, 0x05, 0x4a, 0x4f, 0xac, + 0xd9, 0xe7, 0xa0, 0xe8, 0x60, 0xdb, 0x52, 0x3d, 0xad, 0xef, 0x5f, 0x6d, 0x26, 0x84, 0x7d, 0xad, + 0xef, 0xd2, 0x9b, 0xd9, 0xe4, 0xa3, 0x6e, 0xf4, 0xb1, 0xeb, 0x89, 0xfb, 0xcd, 0x25, 0x42, 0xdb, + 0x61, 0x24, 0xe2, 0x24, 0xd7, 0xf8, 0x7d, 0x56, 0x77, 0xce, 0x2b, 0xf4, 0x19, 0x6d, 0xb1, 0xdb, + 0x76, 0xd3, 0x60, 0xef, 0xf4, 0x2e, 0x5e, 0x1d, 0x0a, 0x31, 0xb8, 0xdd, 0x7f, 0x97, 0x77, 0x01, + 0x85, 0xbd, 0xc0, 0xfd, 0x7d, 0x03, 0x16, 0xa9, 0x93, 0x44, 0x75, 0xb4, 0x9e, 0xe1, 0x06, 0x85, + 0xb3, 0xc9, 0x1a, 0x20, 0xe6, 0xe0, 0x48, 0x45, 0x34, 0xfb, 0xa8, 0x8c, 0xa9, 0x90, 0xfe, 0x4e, + 0x82, 0x53, 0x91, 0x36, 0xb8, 0xad, 0xd7, 0xa3, 0x8d, 0x64, 0x9a, 0xca, 0x1b, 0x68, 0x46, 0x96, + 0x84, 0x1b, 0x59, 0x26, 0xfd, 0x9a, 0x96, 0x83, 0x7f, 0x90, 0x00, 0x1a, 0x23, 0xef, 0x98, 0x23, + 0x83, 0xe1, 0x91, 0x91, 0xa2, 0x23, 0x43, 0xbe, 0xd9, 0x9a, 0xeb, 0xbe, 0xb4, 0x1c, 0xb1, 0xa7, + 0xf1, 0xdf, 0x29, 0x86, 0x37, 0xf2, 0x8e, 0xc5, 0x99, 0x19, 0x79, 0x46, 0x17, 0xa1, 0xc2, 0xae, + 0xd3, 0xab, 0x9a, 0xae, 0x3b, 0xd8, 0x75, 0xf9, 0xe1, 0x59, 0x99, 0x51, 0x1b, 0x8c, 0x48, 0xd8, + 0x0c, 0x8a, 0x6a, 0x7b, 0x27, 0xaa, 0x67, 0x3d, 0xc7, 0x26, 0xdf, 0x9b, 0x94, 0x05, 0x75, 0x9f, + 0x10, 0xd9, 0x29, 0x42, 0xdf, 0x70, 0x3d, 0x47, 0xb0, 0x89, 0x83, 0x1a, 0x4e, 0xa5, 0x6c, 0x64, + 0x50, 0xaa, 0x9d, 0xd1, 0x60, 0xc0, 0x5c, 0xfc, 0xfa, 0xc3, 0xfe, 0x3e, 0xef, 0x50, 0x2e, 0x2b, + 0xa6, 0x03, 0xa7, 0xf1, 0xee, 0xbe, 0x41, 0x10, 0xe6, 0x7d, 0x58, 0x0d, 0xf5, 0x81, 0x87, 0x55, + 0xa4, 0x88, 0x94, 0xa2, 0x45, 0xa4, 0xfc, 0x18, 0x10, 0xc3, 0x1d, 0xbe, 0x61, 0xbf, 0xe5, 0xd3, + 0x70, 0x2a, 0xa2, 0x88, 0xaf, 0xc4, 0xd7, 0xa0, 0xcc, 0xaf, 0x44, 0xf1, 0x40, 0x39, 0x0b, 0x05, + 0x92, 0x51, 0x7b, 0x86, 0x2e, 0x0e, 0x54, 0x97, 0x6c, 0x4b, 0x6f, 0x1a, 0xba, 0x23, 0x7f, 0x02, + 0x65, 0x85, 0xb5, 0xc3, 0x79, 0x1f, 0x41, 0x85, 0x5f, 0xa0, 0x52, 0x23, 0x37, 0x18, 0xd3, 0x6e, + 0xc8, 0x87, 0x1b, 0x51, 0xca, 0x66, 0xf8, 0x55, 0xd6, 0xa1, 0xce, 0x4a, 0x86, 0x88, 0x7a, 0xd1, + 0xd9, 0x47, 0x20, 0x6e, 0x0c, 0x4c, 0x6c, 0x25, 0x2a, 0x5f, 0x76, 0xc2, 0xaf, 0xf2, 0x79, 0x38, + 0x97, 0xda, 0x0a, 0xf7, 0x84, 0x0d, 0xd5, 0xe0, 0x83, 0x6e, 0x88, 0x93, 0x65, 0x7a, 0x62, 0x2c, + 0x85, 0x4e, 0x8c, 0xcf, 0xf8, 0x45, 0x62, 0x4e, 0x2c, 0x62, 0xb4, 0x02, 0x0c, 0xca, 0xfd, 0x7c, + 0x56, 0xb9, 0x3f, 0x1f, 0x29, 0xf7, 0xe5, 0xae, 0xef, 0x4f, 0xbe, 0x0d, 0x7b, 0x48, 0xb7, 0x8b, + 0xac, 0x6d, 0x91, 0x10, 0xe5, 0x71, 0xbd, 0x64, 0xac, 0x4a, 0x48, 0x4a, 0xbe, 0x0a, 0xe5, 0x68, + 0x6a, 0x0c, 0xe5, 0x39, 0x29, 0x91, 0xe7, 0x2a, 0xb1, 0x14, 0xf7, 0x61, 0xac, 0x02, 0xce, 0xf6, + 0x71, 0xac, 0xfe, 0xbd, 0x17, 0x49, 0x76, 0xd7, 0x52, 0x0e, 0x7b, 0x7f, 0x4d, 0x79, 0x6e, 0x8d, + 0xaf, 0x07, 0x8f, 0x5c, 0x22, 0xcf, 0x3b, 0x2d, 0xbf, 0x03, 0xa5, 0x83, 0xac, 0x9f, 0x5f, 0xcc, + 0x8b, 0x8b, 0x15, 0xb7, 0x60, 0xed, 0x91, 0x31, 0xc0, 0xee, 0x89, 0xeb, 0xe1, 0x61, 0x8b, 0x26, + 0xa5, 0x23, 0x03, 0x3b, 0x68, 0x03, 0x80, 0x6e, 0x61, 0x6c, 0xcb, 0xf0, 0x6f, 0xe5, 0x87, 0x28, + 0xf2, 0x7f, 0x4a, 0xb0, 0x12, 0x08, 0x1e, 0xd0, 0xad, 0xdb, 0x5b, 0x50, 0x24, 0xfd, 0x75, 0x3d, + 0x6d, 0x68, 0x8b, 0xf3, 0x2c, 0x9f, 0x80, 0xee, 0xc2, 0xc2, 0x91, 0x2b, 0x20, 0xa3, 0x54, 0x00, + 0x3d, 0xcd, 0x10, 0x65, 0xfe, 0xc8, 0x6d, 0xe9, 0xe8, 0x23, 0x80, 0x91, 0x8b, 0x75, 0x7e, 0x86, + 0x95, 0xcf, 0xaa, 0x16, 0x0e, 0xc2, 0x07, 0xe1, 0x44, 0x80, 0xdd, 0xc9, 0xb8, 0x07, 0x25, 0xc3, + 0xb4, 0x74, 0x4c, 0x0f, 0x27, 0x75, 0x8e, 0x2a, 0x4d, 0x10, 0x07, 0x26, 0x71, 0xe0, 0x62, 0x5d, + 0xc6, 0x7c, 0x2d, 0x14, 0xfe, 0xe5, 0x81, 0xd2, 0x86, 0x55, 0x96, 0xb4, 0x8e, 0x7c, 0xc3, 0x45, + 0xc4, 0x6e, 0x8e, 0xeb, 0x1d, 0xf5, 0x96, 0x52, 0x35, 0x78, 0x69, 0x23, 0x44, 0xe5, 0x3b, 0x70, + 0x3a, 0xb2, 0x43, 0x9a, 0x61, 0xcb, 0x22, 0x77, 0x62, 0x40, 0x49, 0x10, 0xce, 0x1c, 0x86, 0x10, + 0xd1, 0x3c, 0x09, 0x86, 0x70, 0x19, 0x0c, 0xe1, 0xca, 0x9f, 0xc3, 0xd9, 0x08, 0xa2, 0x13, 0xb1, + 0xe8, 0x5e, 0xac, 0x72, 0xbb, 0x34, 0x49, 0x6b, 0xac, 0x84, 0xfb, 0x1f, 0x09, 0xd6, 0xd2, 0x18, + 0x5e, 0x13, 0x71, 0xfc, 0x51, 0xc6, 0x45, 0xbd, 0xdb, 0xd3, 0x99, 0xf5, 0x1b, 0x41, 0x6b, 0xf7, + 0xa1, 0x9e, 0xe6, 0xcf, 0xe4, 0x28, 0xe5, 0x67, 0x19, 0xa5, 0x9f, 0xe5, 0x43, 0xc8, 0x7b, 0xc3, + 0xf3, 0x1c, 0xe3, 0x70, 0x44, 0x42, 0xfe, 0x8d, 0xa3, 0x59, 0x2d, 0x1f, 0x97, 0x61, 0xae, 0xbd, + 0x39, 0x46, 0x3c, 0xb0, 0x23, 0x15, 0x9b, 0xf9, 0x34, 0x8a, 0xcd, 0x30, 0x4c, 0xfd, 0xd6, 0x74, + 0xfa, 0x7e, 0x6b, 0x01, 0xd0, 0x9f, 0xe5, 0xa0, 0x12, 0x1d, 0x22, 0xb4, 0x0b, 0xa0, 0xf9, 0x96, + 0xf3, 0x89, 0x72, 0x71, 0xaa, 0x6e, 0x2a, 0x21, 0x41, 0xf4, 0x1e, 0xe4, 0x7b, 0xf6, 0x88, 0x8f, + 0x5a, 0xca, 0x61, 0x70, 0xd3, 0x1e, 0xb1, 0x8c, 0x42, 0xd8, 0xc8, 0x9e, 0x8a, 0x9d, 0xed, 0x67, + 0x67, 0xc9, 0x67, 0xf4, 0x3b, 0x93, 0xe1, 0xcc, 0xe8, 0x09, 0x54, 0x5e, 0x3a, 0x86, 0xa7, 0x1d, + 0x0e, 0xb0, 0x3a, 0xd0, 0x4e, 0xb0, 0xc3, 0xb3, 0xe4, 0x14, 0x89, 0xac, 0x2c, 0x04, 0x9f, 0x12, + 0x39, 0xf9, 0x0f, 0xa1, 0x20, 0x2c, 0x9a, 0xb0, 0x22, 0xec, 0xc3, 0xfa, 0x88, 0xb0, 0xa9, 0xf4, + 0xae, 0x9c, 0xa9, 0x99, 0x96, 0xea, 0x62, 0xb2, 0x8c, 0x8b, 0xdf, 0x05, 0x4c, 0x48, 0xd1, 0x6b, + 0x54, 0xba, 0x69, 0x39, 0xb8, 0xad, 0x99, 0x56, 0x97, 0x89, 0xca, 0x2f, 0xa0, 0x14, 0xea, 0xe0, + 0x04, 0x13, 0x5a, 0xb0, 0x2a, 0x8e, 0xe2, 0x5d, 0xec, 0xf1, 0xe5, 0x65, 0xaa, 0xc6, 0x57, 0xb8, + 0x5c, 0x17, 0x7b, 0xec, 0xfa, 0xc4, 0x3d, 0x38, 0xab, 0x60, 0xcb, 0xc6, 0xa6, 0x3f, 0x9e, 0x4f, + 0xad, 0xfe, 0x0c, 0x19, 0xfc, 0x2d, 0xa8, 0xa7, 0xc9, 0xb3, 0xfc, 0x70, 0xed, 0x12, 0x14, 0xc4, + 0x6f, 0x69, 0xd1, 0x12, 0xe4, 0xf7, 0x9b, 0x9d, 0xea, 0x1c, 0x79, 0x38, 0xd8, 0xe9, 0x54, 0x25, + 0x54, 0x80, 0xf9, 0x6e, 0x73, 0xbf, 0x53, 0xcd, 0x5d, 0x1b, 0x42, 0x35, 0xfe, 0x43, 0x52, 0xb4, + 0x0e, 0xa7, 0x3a, 0xca, 0x5e, 0xa7, 0xf1, 0xb8, 0xb1, 0xdf, 0xda, 0x6b, 0xab, 0x1d, 0xa5, 0xf5, + 0x71, 0x63, 0x7f, 0xb7, 0x3a, 0x87, 0x36, 0xe1, 0x7c, 0xf8, 0xc3, 0x93, 0xbd, 0xee, 0xbe, 0xba, + 0xbf, 0xa7, 0x36, 0xf7, 0xda, 0xfb, 0x8d, 0x56, 0x7b, 0x57, 0xa9, 0x4a, 0xe8, 0x3c, 0x9c, 0x0d, + 0xb3, 0x3c, 0x6c, 0xed, 0xb4, 0x94, 0xdd, 0x26, 0x79, 0x6e, 0x3c, 0xad, 0xe6, 0xae, 0xdd, 0x84, + 0x72, 0xe4, 0x77, 0x9f, 0xc4, 0xa4, 0xce, 0xde, 0x4e, 0x75, 0x0e, 0x95, 0xa1, 0x18, 0xd6, 0x53, + 0x80, 0xf9, 0xf6, 0xde, 0xce, 0x6e, 0x35, 0x77, 0xed, 0x0e, 0xac, 0xc4, 0xee, 0xf7, 0xa2, 0x55, + 0x28, 0x77, 0x1b, 0xed, 0x9d, 0x87, 0x7b, 0x9f, 0xaa, 0xca, 0x6e, 0x63, 0xe7, 0xb3, 0xea, 0x1c, + 0x5a, 0x83, 0xaa, 0x20, 0xb5, 0xf7, 0xf6, 0x19, 0x55, 0xba, 0xf6, 0x3c, 0x36, 0xc7, 0x30, 0x3a, + 0x0d, 0xab, 0x7e, 0x33, 0x6a, 0x53, 0xd9, 0x6d, 0xec, 0xef, 0x92, 0xd6, 0x23, 0x64, 0xe5, 0xa0, + 0xdd, 0x6e, 0xb5, 0x1f, 0x57, 0x25, 0xa2, 0x35, 0x20, 0xef, 0x7e, 0xda, 0x22, 0xcc, 0xb9, 0x28, + 0xf3, 0x41, 0xfb, 0x07, 0xed, 0xbd, 0x4f, 0xda, 0xd5, 0xfc, 0xf6, 0x2f, 0x56, 0xa1, 0x22, 0x0a, + 0x3d, 0xec, 0xd0, 0x5b, 0x2d, 0x1d, 0x58, 0x12, 0xbf, 0xcd, 0x4e, 0xc9, 0xd0, 0xd1, 0x5f, 0x94, + 0xd7, 0x37, 0xc7, 0x70, 0xf0, 0x7a, 0x7b, 0x0e, 0x1d, 0xd2, 0xfa, 0x37, 0x74, 0xdf, 0xfa, 0x52, + 0x6a, 0xb5, 0x99, 0xb8, 0xe2, 0x5d, 0xbf, 0x3c, 0x91, 0xcf, 0x6f, 0x03, 0x93, 0x12, 0x37, 0xfc, + 0xcb, 0x23, 0x74, 0x39, 0xad, 0x36, 0x4d, 0xf9, 0x69, 0x53, 0xfd, 0xca, 0x64, 0x46, 0xbf, 0x99, + 0xe7, 0x50, 0x8d, 0xff, 0x0a, 0x09, 0xa5, 0x40, 0xa7, 0x19, 0x3f, 0x75, 0xaa, 0x5f, 0x9b, 0x86, + 0x35, 0xdc, 0x58, 0xe2, 0xf7, 0x3a, 0x57, 0xa7, 0xf9, 0x5d, 0x43, 0x66, 0x63, 0x59, 0x3f, 0x81, + 0x60, 0x0e, 0x8c, 0x5e, 0x91, 0x46, 0xa9, 0x3f, 0x8e, 0x49, 0xb9, 0x89, 0x9f, 0xe6, 0xc0, 0xf4, + 0xdb, 0xd6, 0xf2, 0x1c, 0x3a, 0x86, 0x95, 0xd8, 0xf5, 0x04, 0x94, 0x22, 0x9e, 0x7e, 0x0f, 0xa3, + 0x7e, 0x75, 0x0a, 0xce, 0x68, 0x44, 0x84, 0xaf, 0x23, 0xa4, 0x47, 0x44, 0xca, 0x65, 0x87, 0xf4, + 0x88, 0x48, 0xbd, 0xd9, 0x40, 0x83, 0x3b, 0x72, 0x0d, 0x21, 0x2d, 0xb8, 0xd3, 0x2e, 0x3f, 0xd4, + 0x2f, 0x4f, 0xe4, 0x0b, 0x3b, 0x2d, 0x76, 0x29, 0x21, 0xcd, 0x69, 0xe9, 0x97, 0x1e, 0xea, 0x57, + 0xa7, 0xe0, 0x8c, 0x47, 0x41, 0x70, 0xc4, 0x99, 0x15, 0x05, 0x89, 0x03, 0xf9, 0xac, 0x28, 0x48, + 0x9e, 0x96, 0xf2, 0x28, 0x88, 0x1d, 0x4d, 0x5e, 0x99, 0xe2, 0x28, 0x25, 0x3b, 0x0a, 0xd2, 0x0f, + 0x5d, 0xe4, 0x39, 0xf4, 0x53, 0x09, 0x6a, 0x59, 0xc7, 0x14, 0x28, 0xa5, 0xbe, 0x9b, 0x70, 0xb2, + 0x52, 0xdf, 0x9e, 0x45, 0xc4, 0xb7, 0xe2, 0x4b, 0x40, 0xc9, 0x75, 0x0f, 0x7d, 0x27, 0x6d, 0x64, + 0x32, 0x56, 0xd7, 0xfa, 0x7b, 0xd3, 0x31, 0xfb, 0x4d, 0x76, 0xa1, 0x20, 0x0e, 0x46, 0x50, 0x4a, + 0x96, 0x8e, 0x1d, 0xcb, 0xd4, 0xe5, 0x71, 0x2c, 0xbe, 0xd2, 0xc7, 0x30, 0x4f, 0xa8, 0xe8, 0x7c, + 0x3a, 0xb7, 0x50, 0xb6, 0x91, 0xf5, 0xd9, 0x57, 0xf4, 0x0c, 0x16, 0xd9, 0x49, 0x00, 0x4a, 0x41, + 0x1e, 0x22, 0xe7, 0x15, 0xf5, 0x0b, 0xd9, 0x0c, 0xbe, 0xba, 0x2f, 0xd8, 0xbf, 0xed, 0xe0, 0x20, + 0x3f, 0x7a, 0x37, 0xfd, 0x77, 0xd0, 0xd1, 0x33, 0x85, 0xfa, 0xc5, 0x09, 0x5c, 0xe1, 0x49, 0x11, + 0xab, 0x7a, 0x2f, 0x4f, 0xdc, 0xba, 0x64, 0x4f, 0x8a, 0xf4, 0xcd, 0x11, 0x0b, 0x92, 0xe4, 0xe6, + 0x29, 0x2d, 0x48, 0x32, 0xb7, 0xac, 0x69, 0x41, 0x92, 0xbd, 0x1f, 0x93, 0xe7, 0x90, 0x07, 0xa7, + 0x52, 0xa0, 0x32, 0xf4, 0x5e, 0x56, 0x90, 0xa7, 0xe1, 0x76, 0xf5, 0xeb, 0x53, 0x72, 0x87, 0x07, + 0x9f, 0x4f, 0xfa, 0xb7, 0xb3, 0xf1, 0xa3, 0xcc, 0xc1, 0x8f, 0x4f, 0xf1, 0xed, 0x7f, 0xc9, 0xc3, + 0x32, 0x83, 0x41, 0x79, 0x05, 0xf3, 0x19, 0x40, 0x70, 0x02, 0x81, 0xde, 0x49, 0xf7, 0x49, 0xe4, + 0x94, 0xa6, 0xfe, 0xee, 0x78, 0xa6, 0x70, 0xa0, 0x85, 0xd0, 0xfc, 0xb4, 0x40, 0x4b, 0x1e, 0x5a, + 0xa4, 0x05, 0x5a, 0xca, 0x91, 0x80, 0x3c, 0x87, 0x3e, 0x86, 0xa2, 0x0f, 0x1b, 0xa3, 0x34, 0xd8, + 0x39, 0x86, 0x8b, 0xd7, 0xdf, 0x19, 0xcb, 0x13, 0xb6, 0x3a, 0x84, 0x09, 0xa7, 0x59, 0x9d, 0xc4, + 0x9e, 0xd3, 0xac, 0x4e, 0x03, 0x96, 0x03, 0x9f, 0x30, 0xe4, 0x28, 0xd3, 0x27, 0x11, 0xe0, 0x2e, + 0xd3, 0x27, 0x51, 0xf8, 0x49, 0x9e, 0x7b, 0x78, 0xe9, 0x97, 0x5f, 0x6d, 0x48, 0xff, 0xf4, 0xd5, + 0xc6, 0xdc, 0x4f, 0xbe, 0xde, 0x90, 0x7e, 0xf9, 0xf5, 0x86, 0xf4, 0x8f, 0x5f, 0x6f, 0x48, 0xff, + 0xfa, 0xf5, 0x86, 0xf4, 0xa7, 0xff, 0xb6, 0x31, 0xf7, 0xc3, 0x82, 0x90, 0x3e, 0x5c, 0xa4, 0xff, + 0x7c, 0xe7, 0x83, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xaa, 0xb3, 0xf4, 0xf2, 0x42, 0x49, 0x00, + 0x00, } diff --git a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto index c2df37b8045..4564de5e3c3 100644 --- a/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto +++ b/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto @@ -637,6 +637,9 @@ message WindowsContainerSecurityContext { // exist in the container image and be resolved there by the runtime; // otherwise, the runtime MUST return error. string run_as_username = 1; + + // The contents of the GMSA credential spec to use to run this container. + string credential_spec = 2; } // WindowsContainerConfig contains platform-specific configuration for diff --git a/staging/src/k8s.io/csi-translation-lib/go.sum b/staging/src/k8s.io/csi-translation-lib/go.sum index 71180aad28e..fa1c4c6126d 100644 --- a/staging/src/k8s.io/csi-translation-lib/go.sum +++ b/staging/src/k8s.io/csi-translation-lib/go.sum @@ -63,8 +63,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/staging/src/k8s.io/kube-aggregator/go.mod b/staging/src/k8s.io/kube-aggregator/go.mod index d5cfc6d1e94..4ed030a8eba 100644 --- a/staging/src/k8s.io/kube-aggregator/go.mod +++ b/staging/src/k8s.io/kube-aggregator/go.mod @@ -21,7 +21,7 @@ require ( k8s.io/client-go v0.0.0 k8s.io/code-generator v0.0.0 k8s.io/component-base v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 k8s.io/utils v0.0.0-20190221042446-c2654d5206da ) diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index 41a22c8c14a..decb6744b22 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -208,8 +208,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/kube-controller-manager/go.sum b/staging/src/k8s.io/kube-controller-manager/go.sum index f0923677eb0..dd078baa515 100644 --- a/staging/src/k8s.io/kube-controller-manager/go.sum +++ b/staging/src/k8s.io/kube-controller-manager/go.sum @@ -56,8 +56,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= diff --git a/staging/src/k8s.io/kube-proxy/go.sum b/staging/src/k8s.io/kube-proxy/go.sum index f0923677eb0..dd078baa515 100644 --- a/staging/src/k8s.io/kube-proxy/go.sum +++ b/staging/src/k8s.io/kube-proxy/go.sum @@ -56,8 +56,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= diff --git a/staging/src/k8s.io/kube-scheduler/go.sum b/staging/src/k8s.io/kube-scheduler/go.sum index f0923677eb0..dd078baa515 100644 --- a/staging/src/k8s.io/kube-scheduler/go.sum +++ b/staging/src/k8s.io/kube-scheduler/go.sum @@ -56,8 +56,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= diff --git a/staging/src/k8s.io/kubelet/go.sum b/staging/src/k8s.io/kubelet/go.sum index 44ad19d7d2f..53da16484ca 100644 --- a/staging/src/k8s.io/kubelet/go.sum +++ b/staging/src/k8s.io/kubelet/go.sum @@ -48,8 +48,8 @@ gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD b/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD index 127b74bc54a..3fa4d5b4a49 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD @@ -82,7 +82,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/cloud-provider/volume:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go index f7c05545895..07f0bc064cb 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go @@ -205,16 +205,6 @@ const volumeAttachmentStuck = "VolumeAttachmentStuck" // Indicates that a node has volumes stuck in attaching state and hence it is not fit for scheduling more pods const nodeWithImpairedVolumes = "NodeWithImpairedVolumes" -const ( - // These constants help to identify if a node is a master or a minion - labelKeyNodeRole = "kubernetes.io/role" - nodeMasterRole = "master" - nodeMinionRole = "node" - labelKeyNodeMaster = "node-role.kubernetes.io/master" - labelKeyNodeCompute = "node-role.kubernetes.io/compute" - labelKeyNodeMinion = "node-role.kubernetes.io/node" -) - const ( // volumeAttachmentConsecutiveErrorLimit is the number of consecutive errors we will ignore when waiting for a volume to attach/detach volumeAttachmentStatusConsecutiveErrorLimit = 10 @@ -1628,30 +1618,64 @@ func (c *Cloud) InstanceType(ctx context.Context, nodeName types.NodeName) (stri // GetCandidateZonesForDynamicVolume retrieves a list of all the zones in which nodes are running // It currently involves querying all instances func (c *Cloud) GetCandidateZonesForDynamicVolume() (sets.String, error) { - zones := sets.NewString() + // We don't currently cache this; it is currently used only in volume + // creation which is expected to be a comparatively rare occurrence. - // TODO: list from cache? - nodes, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{}) + // TODO: Caching / expose v1.Nodes to the cloud provider? + // TODO: We could also query for subnets, I think + + // Note: It is more efficient to call the EC2 API twice with different tag + // filters than to call it once with a tag filter that results in a logical + // OR. For really large clusters the logical OR will result in EC2 API rate + // limiting. + instances := []*ec2.Instance{} + + baseFilters := []*ec2.Filter{newEc2Filter("instance-state-name", "running")} + + filters := c.tagging.addFilters(baseFilters) + di, err := c.describeInstances(filters) if err != nil { - klog.Errorf("Failed to get nodes from api server: %#v", err) return nil, err } - for _, n := range nodes.Items { - if !c.isNodeReady(&n) { - klog.V(4).Infof("Ignoring not ready node %q in zone discovery", n.Name) + instances = append(instances, di...) + + if c.tagging.usesLegacyTags { + filters = c.tagging.addLegacyFilters(baseFilters) + di, err = c.describeInstances(filters) + if err != nil { + return nil, err + } + + instances = append(instances, di...) + } + + if len(instances) == 0 { + return nil, fmt.Errorf("no instances returned") + } + + zones := sets.NewString() + + for _, instance := range instances { + // We skip over master nodes, if the installation tool labels them with one of the well-known master labels + // This avoids creating a volume in a zone where only the master is running - e.g. #34583 + // This is a short-term workaround until the scheduler takes care of zone selection + master := false + for _, tag := range instance.Tags { + tagKey := aws.StringValue(tag.Key) + if awsTagNameMasterRoles.Has(tagKey) { + master = true + } + } + + if master { + klog.V(4).Infof("Ignoring master instance %q in zone discovery", aws.StringValue(instance.InstanceId)) continue } - // In some cluster provisioning software, a node can be both a minion and a master. Therefore we white-list - // here, and only filter out node that is not minion AND is labeled as master explicitly - if c.isMinionNode(&n) || !c.isMasterNode(&n) { - if zone, ok := n.Labels[v1.LabelZoneFailureDomain]; ok { - zones.Insert(zone) - } else { - klog.Warningf("Node %s does not have zone label, ignore for zone discovery.", n.Name) - } - } else { - klog.V(4).Infof("Ignoring master node %q in zone discovery", n.Name) + + if instance.Placement != nil { + zone := aws.StringValue(instance.Placement.AvailabilityZone) + zones.Insert(zone) } } @@ -1659,48 +1683,6 @@ func (c *Cloud) GetCandidateZonesForDynamicVolume() (sets.String, error) { return zones, nil } -// isNodeReady checks node condition and return true if NodeReady is marked as true -func (c *Cloud) isNodeReady(node *v1.Node) bool { - for _, c := range node.Status.Conditions { - if c.Type == v1.NodeReady { - return c.Status == v1.ConditionTrue - } - } - return false -} - -// isMasterNode checks if the node is labeled as master -func (c *Cloud) isMasterNode(node *v1.Node) bool { - // Master node has one or more of the following labels: - // - // kubernetes.io/role: master - // node-role.kubernetes.io/master: "" - // node-role.kubernetes.io/master: "true" - if val, ok := node.Labels[labelKeyNodeMaster]; ok && val != "false" { - return true - } else if role, ok := node.Labels[labelKeyNodeRole]; ok && role == nodeMasterRole { - return true - } - return false -} - -// isMinionNode checks if the node is labeled as minion -func (c *Cloud) isMinionNode(node *v1.Node) bool { - // Minion node has one or more oof the following labels: - // - // kubernetes.io/role: "node" - // node-role.kubernetes.io/compute: "true" - // node-role.kubernetes.io/node: "" - if val, ok := node.Labels[labelKeyNodeMinion]; ok && val != "false" { - return true - } else if val, ok := node.Labels[labelKeyNodeCompute]; ok && val != "false" { - return true - } else if role, ok := node.Labels[labelKeyNodeRole]; ok && role == nodeMinionRole { - return true - } - return false -} - // GetZone implements Zones.GetZone func (c *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { return cloudprovider.Zone{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go index 8f9216241ee..ac1d311c52a 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go @@ -36,7 +36,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/kubernetes/fake" cloudvolume "k8s.io/cloud-provider/volume" ) @@ -1876,118 +1875,6 @@ func TestRegionIsValid(t *testing.T) { assert.False(t, isRegionValid("pl-fake-991a", fake.metadata), "expected region 'pl-fake-991' to be invalid but it was not") } -func TestGetCandidateZonesForDynamicVolume(t *testing.T) { - tests := []struct { - name string - labels map[string]string - ready bool - expectedZones sets.String - }{ - { - name: "master node with role label", - labels: map[string]string{labelKeyNodeRole: nodeMasterRole, v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString(), - }, - { - name: "master node with master label empty", - labels: map[string]string{labelKeyNodeMaster: "", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString(), - }, - { - name: "master node with master label true", - labels: map[string]string{labelKeyNodeMaster: "true", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString(), - }, - { - name: "master node with master label false", - labels: map[string]string{labelKeyNodeMaster: "false", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - { - name: "minion node with role label", - labels: map[string]string{labelKeyNodeRole: nodeMinionRole, v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - { - name: "minion node with minion label", - labels: map[string]string{labelKeyNodeMinion: "", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - { - name: "minion node with compute label", - labels: map[string]string{labelKeyNodeCompute: "true", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - { - name: "master and minion node", - labels: map[string]string{labelKeyNodeMaster: "true", labelKeyNodeCompute: "true", v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - { - name: "node not ready", - labels: map[string]string{v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: false, - expectedZones: sets.NewString(), - }, - { - name: "node has no zone", - labels: map[string]string{}, - ready: true, - expectedZones: sets.NewString(), - }, - { - name: "node with no label", - labels: map[string]string{v1.LabelZoneFailureDomain: "us-east-1a"}, - ready: true, - expectedZones: sets.NewString("us-east-1a"), - }, - } - - for i, test := range tests { - t.Run(test.name, func(t *testing.T) { - awsServices := newMockedFakeAWSServices(TestClusterID) - c, _ := newAWSCloud(CloudConfig{}, awsServices) - c.kubeClient = fake.NewSimpleClientset() - nodeName := fmt.Sprintf("node-%d", i) - _, err := c.kubeClient.CoreV1().Nodes().Create(&v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Labels: test.labels, - }, - Status: genNodeStatus(test.ready), - }) - assert.Nil(t, err) - zones, err := c.GetCandidateZonesForDynamicVolume() - assert.Nil(t, err) - assert.Equal(t, test.expectedZones, zones) - }) - } -} - -func genNodeStatus(ready bool) v1.NodeStatus { - status := v1.ConditionFalse - if ready { - status = v1.ConditionTrue - } - ret := v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeReady, - Status: status, - }, - }, - } - return ret -} - func newMockedFakeAWSServices(id string) *FakeAWSServices { s := NewFakeAWSServices(id) s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go index a07da82a73c..281ba764363 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go @@ -596,8 +596,9 @@ func shouldRetryHTTPRequest(resp *http.Response, err error) bool { return false } +// processHTTPRetryResponse : return true means stop retry, false means continue retry func (az *Cloud) processHTTPRetryResponse(service *v1.Service, reason string, resp *http.Response, err error) (bool, error) { - if resp != nil && isSuccessHTTPResponse(resp) { + if err == nil && resp != nil && isSuccessHTTPResponse(resp) { // HTTP 2xx suggests a successful response return true, nil } @@ -620,7 +621,7 @@ func (az *Cloud) processHTTPRetryResponse(service *v1.Service, reason string, re } func (az *Cloud) processHTTPResponse(service *v1.Service, reason string, resp *http.Response, err error) error { - if isSuccessHTTPResponse(resp) { + if err == nil && isSuccessHTTPResponse(resp) { // HTTP 2xx suggests a successful response return nil } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go index d0a1399e27c..c14ace656f2 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go @@ -119,6 +119,11 @@ func TestProcessRetryResponse(t *testing.T) { code: http.StatusOK, stop: true, }, + { + code: http.StatusOK, + err: fmt.Errorf("some error"), + stop: false, + }, { code: 399, stop: true, diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.mod b/staging/src/k8s.io/legacy-cloud-providers/go.mod index fd4e0953084..61b90c135fc 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.mod +++ b/staging/src/k8s.io/legacy-cloud-providers/go.mod @@ -28,7 +28,7 @@ require ( k8s.io/client-go v0.0.0 k8s.io/cloud-provider v0.0.0 k8s.io/csi-translation-lib v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 k8s.io/utils v0.0.0-20190221042446-c2654d5206da sigs.k8s.io/yaml v1.1.0 ) diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.sum b/staging/src/k8s.io/legacy-cloud-providers/go.sum index 66ef7b4cff1..8e6a72c3505 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.sum +++ b/staging/src/k8s.io/legacy-cloud-providers/go.sum @@ -123,8 +123,8 @@ gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/metrics/go.sum b/staging/src/k8s.io/metrics/go.sum index 12f4f6d1ec1..b3a4f260ef3 100644 --- a/staging/src/k8s.io/metrics/go.sum +++ b/staging/src/k8s.io/metrics/go.sum @@ -93,8 +93,8 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/node-api/go.sum b/staging/src/k8s.io/node-api/go.sum index f08159a0702..355a93899a1 100644 --- a/staging/src/k8s.io/node-api/go.sum +++ b/staging/src/k8s.io/node-api/go.sum @@ -95,8 +95,8 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/sample-apiserver/go.mod b/staging/src/k8s.io/sample-apiserver/go.mod index aafff806994..9f19dd33203 100644 --- a/staging/src/k8s.io/sample-apiserver/go.mod +++ b/staging/src/k8s.io/sample-apiserver/go.mod @@ -13,7 +13,7 @@ require ( k8s.io/client-go v0.0.0 k8s.io/code-generator v0.0.0 k8s.io/component-base v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 ) replace ( diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index 3830d03eeac..1505b366188 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -205,8 +205,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/sample-cli-plugin/go.sum b/staging/src/k8s.io/sample-cli-plugin/go.sum index 82c5e91fad2..10482de0a13 100644 --- a/staging/src/k8s.io/sample-cli-plugin/go.sum +++ b/staging/src/k8s.io/sample-cli-plugin/go.sum @@ -110,8 +110,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/staging/src/k8s.io/sample-controller/go.mod b/staging/src/k8s.io/sample-controller/go.mod index 71c19564e91..813e48f3b3d 100644 --- a/staging/src/k8s.io/sample-controller/go.mod +++ b/staging/src/k8s.io/sample-controller/go.mod @@ -9,7 +9,7 @@ require ( k8s.io/apimachinery v0.0.0 k8s.io/client-go v0.0.0 k8s.io/code-generator v0.0.0 - k8s.io/klog v0.3.0 + k8s.io/klog v0.3.1 ) replace ( diff --git a/staging/src/k8s.io/sample-controller/go.sum b/staging/src/k8s.io/sample-controller/go.sum index 1a93cf22765..3028ee4279e 100644 --- a/staging/src/k8s.io/sample-controller/go.sum +++ b/staging/src/k8s.io/sample-controller/go.sum @@ -96,8 +96,8 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af h1:SwjZbO0u5ZuaV6TRMWOGB40iaycX8sbdMQHtjNZ19dk= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4= diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index 3cf73f9fed4..07f2d3dbc35 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -40,6 +40,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + // ensure libs have a chance to initialize _ "github.com/stretchr/testify/assert" ) @@ -107,6 +108,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh ginkgo.It("Should be able to convert from CR v1 to CR v2", func() { testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + crd.Spec.Versions = apiVersions crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ Strategy: v1beta1.WebhookConverter, WebhookClientConfig: &v1beta1.WebhookClientConfig{ @@ -129,6 +131,7 @@ var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebh ginkgo.It("Should be able to convert a non homogeneous list of CRs", func() { testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *v1beta1.CustomResourceDefinition) { + crd.Spec.Versions = apiVersions crd.Spec.Conversion = &v1beta1.CustomResourceConversion{ Strategy: v1beta1.WebhookConverter, WebhookClientConfig: &v1beta1.WebhookClientConfig{ @@ -384,6 +387,8 @@ func testCRListConversion(f *framework.Framework, testCrd *crd.TestCrd) { // After changing a CRD, the resources for versions will be re-created that can be result in // cancelled connection (e.g. "grpc connection closed" or "context canceled"). // Just retrying fixes that. + // + // TODO: we have to wait for the storage version to become effective. Storage version changes are not instant. for i := 0; i < 5; i++ { _, err = customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) if err == nil { diff --git a/test/e2e/apimachinery/crd_publish_openapi.go b/test/e2e/apimachinery/crd_publish_openapi.go index 7f3e4e69f2e..04766e40261 100644 --- a/test/e2e/apimachinery/crd_publish_openapi.go +++ b/test/e2e/apimachinery/crd_publish_openapi.go @@ -32,6 +32,7 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" @@ -48,7 +49,7 @@ var ( metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}` ) -var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublishOpenAPI]", func() { +var _ = SIGDescribe("CustomResourcePublishOpenAPI", func() { f := framework.NewDefaultFramework("crd-publish-openapi") ginkgo.BeforeEach(func() { @@ -309,6 +310,10 @@ var _ = SIGDescribe("CustomResourcePublishOpenAPI [Feature:CustomResourcePublish } ginkgo.By("mark a version not serverd") + crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Crd.Name, metav1.GetOptions{}) + if err != nil { + framework.Failf("%v", err) + } crd.Crd.Spec.Versions[1].Served = false crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd.Crd) if err != nil { @@ -336,28 +341,34 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version return nil, fmt.Errorf("require at least one version for CRD") } - if schema == nil { - schema = []byte(`type: object`) - } + expect := schema props := &v1beta1.JSONSchemaProps{} - if err := yaml.Unmarshal(schema, props); err != nil { - return nil, err + if schema == nil { + // to be backwards compatible, we expect CRD controller to treat + // CRD with nil schema specially and publish an empty schema + expect = []byte(`type: object`) + } else { + if err := yaml.Unmarshal(schema, props); err != nil { + return nil, err + } } crd, err := crd.CreateMultiVersionTestCRD(f, group, func(crd *v1beta1.CustomResourceDefinition) { - apiVersions := []v1beta1.CustomResourceDefinitionVersion{} - for _, version := range versions { - v := v1beta1.CustomResourceDefinitionVersion{ + var apiVersions []v1beta1.CustomResourceDefinitionVersion + for i, version := range versions { + apiVersions = append(apiVersions, v1beta1.CustomResourceDefinitionVersion{ Name: version, Served: true, - Storage: false, - } - apiVersions = append(apiVersions, v) + Storage: i == 0, + }) } - apiVersions[0].Storage = true + crd.Spec.Versions = apiVersions - crd.Spec.Validation = &v1beta1.CustomResourceValidation{ - OpenAPIV3Schema: props, + // set up validation when input schema isn't nil + if schema != nil { + crd.Spec.Validation = &v1beta1.CustomResourceValidation{ + OpenAPIV3Schema: props, + } } }) if err != nil { @@ -365,7 +376,7 @@ func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, version } for _, v := range crd.Crd.Spec.Versions { - if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), schema); err != nil { + if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), expect); err != nil { return nil, fmt.Errorf("%v", err) } } @@ -580,9 +591,13 @@ properties: properties: dummy: description: Dummy property. + type: object status: description: Status of Waldo type: object properties: bars: - description: List of Bars and their statuses.`) + description: List of Bars and their statuses. + type: array + items: + type: object`) diff --git a/test/e2e/autoscaling/autoscaling_timer.go b/test/e2e/autoscaling/autoscaling_timer.go index 6a21f626737..8b8831bc0f5 100644 --- a/test/e2e/autoscaling/autoscaling_timer.go +++ b/test/e2e/autoscaling/autoscaling_timer.go @@ -25,15 +25,15 @@ import ( "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" ) var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling", func() { f := framework.NewDefaultFramework("autoscaling") SIGDescribe("Autoscaling a service", func() { - BeforeEach(func() { + ginkgo.BeforeEach(func() { // Check if Cloud Autoscaler is enabled by trying to get its ConfigMap. _, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Get("cluster-autoscaler-status", metav1.GetOptions{}) if err != nil { @@ -41,12 +41,12 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" } }) - Context("from 1 pod and 3 nodes to 8 pods and >=4 nodes", func() { + ginkgo.Context("from 1 pod and 3 nodes to 8 pods and >=4 nodes", func() { const nodesNum = 3 // Expect there to be 3 nodes before and after the test. var nodeGroupName string // Set by BeforeEach, used by AfterEach to scale this node group down after the test. var nodes *v1.NodeList // Set by BeforeEach, used by Measure to calculate CPU request based on node's sizes. - BeforeEach(func() { + ginkgo.BeforeEach(func() { // Make sure there is only 1 node group, otherwise this test becomes useless. nodeGroups := strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") if len(nodeGroups) != 1 { @@ -64,10 +64,10 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" // Make sure all nodes are schedulable, otherwise we are in some kind of a problem state. nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet) schedulableCount := len(nodes.Items) - Expect(schedulableCount).To(Equal(nodeGroupSize), "not all nodes are schedulable") + gomega.Expect(schedulableCount).To(gomega.Equal(nodeGroupSize), "not all nodes are schedulable") }) - AfterEach(func() { + ginkgo.AfterEach(func() { // Attempt cleanup only if a node group was targeted for scale up. // Otherwise the test was probably skipped and we'll get a gcloud error due to invalid parameters. if len(nodeGroupName) > 0 { @@ -77,7 +77,7 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" } }) - Measure("takes less than 15 minutes", func(b Benchmarker) { + ginkgo.Measure("takes less than 15 minutes", func(b ginkgo.Benchmarker) { // Measured over multiple samples, scaling takes 10 +/- 2 minutes, so 15 minutes should be fully sufficient. const timeToWait = 15 * time.Minute @@ -85,8 +85,8 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" // This test expects that 8 pods will not fit in 'nodesNum' nodes, but will fit in >='nodesNum'+1 nodes. // Make it so that 'nodesNum' pods fit perfectly per node. nodeCpus := nodes.Items[0].Status.Allocatable[v1.ResourceCPU] - nodeCpuMillis := (&nodeCpus).MilliValue() - cpuRequestMillis := int64(nodeCpuMillis / nodesNum) + nodeCPUMillis := (&nodeCpus).MilliValue() + cpuRequestMillis := int64(nodeCPUMillis / nodesNum) // Start the service we want to scale and wait for it to be up and running. nodeMemoryBytes := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] @@ -99,10 +99,10 @@ var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling" // Enable Horizontal Pod Autoscaler with 50% target utilization and // scale up the CPU usage to trigger autoscaling to 8 pods for target to be satisfied. - targetCpuUtilizationPercent := int32(50) - hpa := common.CreateCPUHorizontalPodAutoscaler(resourceConsumer, targetCpuUtilizationPercent, 1, 10) + targetCPUUtilizationPercent := int32(50) + hpa := common.CreateCPUHorizontalPodAutoscaler(resourceConsumer, targetCPUUtilizationPercent, 1, 10) defer common.DeleteHorizontalPodAutoscaler(resourceConsumer, hpa.Name) - cpuLoad := 8 * cpuRequestMillis * int64(targetCpuUtilizationPercent) / 100 // 8 pods utilized to the target level + cpuLoad := 8 * cpuRequestMillis * int64(targetCPUUtilizationPercent) / 100 // 8 pods utilized to the target level resourceConsumer.ConsumeCPU(int(cpuLoad)) // Measure the time it takes for the service to scale to 8 pods with 50% CPU utilization each. diff --git a/test/e2e/autoscaling/cluster_autoscaler_scalability.go b/test/e2e/autoscaling/cluster_autoscaler_scalability.go index fbb2c1edde1..5d88fe23c91 100644 --- a/test/e2e/autoscaling/cluster_autoscaler_scalability.go +++ b/test/e2e/autoscaling/cluster_autoscaler_scalability.go @@ -33,8 +33,8 @@ import ( testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "k8s.io/klog" ) @@ -65,7 +65,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun var originalSizes map[string]int var sum int - BeforeEach(func() { + ginkgo.BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke", "kubemark") // Check if Cloud Autoscaler is enabled by trying to get its ConfigMap. @@ -81,7 +81,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { size, err := framework.GroupSize(mig) framework.ExpectNoError(err) - By(fmt.Sprintf("Initial size of %s: %d", mig, size)) + ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size sum += size } @@ -91,13 +91,13 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) nodeCount = len(nodes.Items) - Expect(nodeCount).NotTo(BeZero()) + gomega.Expect(nodeCount).NotTo(gomega.BeZero()) cpu := nodes.Items[0].Status.Capacity[v1.ResourceCPU] mem := nodes.Items[0].Status.Capacity[v1.ResourceMemory] coresPerNode = int((&cpu).MilliValue() / 1000) memCapacityMb = int((&mem).Value() / 1024 / 1024) - Expect(nodeCount).Should(Equal(sum)) + gomega.Expect(nodeCount).Should(gomega.Equal(sum)) if framework.ProviderIs("gke") { val, err := isAutoscalerEnabled(3) @@ -109,8 +109,8 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun } }) - AfterEach(func() { - By(fmt.Sprintf("Restoring initial size of the cluster")) + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) setMigSizes(originalSizes) framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount, scaleDownTimeout)) nodes, err := c.CoreV1().Nodes().List(metav1.ListOptions{}) @@ -132,7 +132,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) }) - It("should scale up at all [Feature:ClusterAutoscalerScalability1]", func() { + ginkgo.It("should scale up at all [Feature:ClusterAutoscalerScalability1]", func() { perNodeReservation := int(float64(memCapacityMb) * 0.95) replicasPerNode := 10 @@ -155,7 +155,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun defer testCleanup() }) - It("should scale up twice [Feature:ClusterAutoscalerScalability2]", func() { + ginkgo.It("should scale up twice [Feature:ClusterAutoscalerScalability2]", func() { perNodeReservation := int(float64(memCapacityMb) * 0.95) replicasPerNode := 10 additionalNodes1 := int(math.Ceil(0.7 * maxNodes)) @@ -204,7 +204,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun klog.Infof("Scaled up twice") }) - It("should scale down empty nodes [Feature:ClusterAutoscalerScalability3]", func() { + ginkgo.It("should scale down empty nodes [Feature:ClusterAutoscalerScalability3]", func() { perNodeReservation := int(float64(memCapacityMb) * 0.7) replicas := int(math.Ceil(maxNodes * 0.7)) totalNodes := maxNodes @@ -232,7 +232,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun }, scaleDownTimeout)) }) - It("should scale down underutilized nodes [Feature:ClusterAutoscalerScalability4]", func() { + ginkgo.It("should scale down underutilized nodes [Feature:ClusterAutoscalerScalability4]", func() { perPodReservation := int(float64(memCapacityMb) * 0.01) // underutilizedNodes are 10% full underutilizedPerNodeReplicas := 10 @@ -291,7 +291,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun }, timeout)) }) - It("shouldn't scale down with underutilized nodes due to host port conflicts [Feature:ClusterAutoscalerScalability5]", func() { + ginkgo.It("shouldn't scale down with underutilized nodes due to host port conflicts [Feature:ClusterAutoscalerScalability5]", func() { fullReservation := int(float64(memCapacityMb) * 0.9) hostPortPodReservation := int(float64(memCapacityMb) * 0.3) totalNodes := maxNodes @@ -307,28 +307,28 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun fullNodesCount := divider underutilizedNodesCount := totalNodes - fullNodesCount - By("Reserving full nodes") + ginkgo.By("Reserving full nodes") // run RC1 w/o host port cleanup := ReserveMemory(f, "filling-pod", fullNodesCount, fullNodesCount*fullReservation, true, largeScaleUpTimeout*2) defer cleanup() - By("Reserving host ports on remaining nodes") + ginkgo.By("Reserving host ports on remaining nodes") // run RC2 w/ host port cleanup2 := createHostPortPodsWithMemory(f, "underutilizing-host-port-pod", underutilizedNodesCount, reservedPort, underutilizedNodesCount*hostPortPodReservation, largeScaleUpTimeout) defer cleanup2() waitForAllCaPodsReadyInNamespace(f, c) // wait and check scale down doesn't occur - By(fmt.Sprintf("Sleeping %v minutes...", scaleDownTimeout.Minutes())) + ginkgo.By(fmt.Sprintf("Sleeping %v minutes...", scaleDownTimeout.Minutes())) time.Sleep(scaleDownTimeout) - By("Checking if the number of nodes is as expected") + ginkgo.By("Checking if the number of nodes is as expected") nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) klog.Infof("Nodes: %v, expected: %v", len(nodes.Items), totalNodes) - Expect(len(nodes.Items)).Should(Equal(totalNodes)) + gomega.Expect(len(nodes.Items)).Should(gomega.Equal(totalNodes)) }) - Specify("CA ignores unschedulable pods while scheduling schedulable pods [Feature:ClusterAutoscalerScalability6]", func() { + ginkgo.Specify("CA ignores unschedulable pods while scheduling schedulable pods [Feature:ClusterAutoscalerScalability6]", func() { // Start a number of pods saturating existing nodes. perNodeReservation := int(float64(memCapacityMb) * 0.80) replicasPerNode := 10 @@ -348,7 +348,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaler scalability [Slow]", fun defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, podsConfig.Name) // Ensure that no new nodes have been added so far. - Expect(framework.NumberOfReadyNodes(f.ClientSet)).To(Equal(nodeCount)) + gomega.Expect(framework.NumberOfReadyNodes(f.ClientSet)).To(gomega.Equal(nodeCount)) // Start a number of schedulable pods to ensure CA reacts. additionalNodes := maxNodes - nodeCount @@ -375,7 +375,7 @@ func anyKey(input map[string]int) string { func simpleScaleUpTestWithTolerance(f *framework.Framework, config *scaleUpTestConfig, tolerateMissingNodeCount int, tolerateMissingPodCount int) func() error { // resize cluster to start size // run rc based on config - By(fmt.Sprintf("Running RC %v from config", config.extraPods.Name)) + ginkgo.By(fmt.Sprintf("Running RC %v from config", config.extraPods.Name)) start := time.Now() framework.ExpectNoError(framework.RunRC(*config.extraPods)) // check results @@ -461,7 +461,7 @@ func addAnnotation(f *framework.Framework, nodes []v1.Node, key, value string) e } func createHostPortPodsWithMemory(f *framework.Framework, id string, replicas, port, megabytes int, timeout time.Duration) func() error { - By(fmt.Sprintf("Running RC which reserves host port and memory")) + ginkgo.By(fmt.Sprintf("Running RC which reserves host port and memory")) request := int64(1024 * 1024 * megabytes / replicas) config := &testutils.RCConfig{ Client: f.ClientSet, diff --git a/test/e2e/autoscaling/cluster_size_autoscaling.go b/test/e2e/autoscaling/cluster_size_autoscaling.go index 4a604f2e060..f8c96694753 100644 --- a/test/e2e/autoscaling/cluster_size_autoscaling.go +++ b/test/e2e/autoscaling/cluster_size_autoscaling.go @@ -48,8 +48,8 @@ import ( testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "k8s.io/klog" ) @@ -94,7 +94,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { var memAllocatableMb int var originalSizes map[string]int - BeforeEach(func() { + ginkgo.BeforeEach(func() { c = f.ClientSet framework.SkipUnlessProviderIs("gce", "gke") @@ -103,7 +103,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { size, err := framework.GroupSize(mig) framework.ExpectNoError(err) - By(fmt.Sprintf("Initial size of %s: %d", mig, size)) + ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size sum += size } @@ -117,12 +117,12 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { quantity := node.Status.Allocatable[v1.ResourceCPU] coreCount += quantity.Value() } - By(fmt.Sprintf("Initial number of schedulable nodes: %v", nodeCount)) - Expect(nodeCount).NotTo(BeZero()) + ginkgo.By(fmt.Sprintf("Initial number of schedulable nodes: %v", nodeCount)) + gomega.Expect(nodeCount).NotTo(gomega.BeZero()) mem := nodes.Items[0].Status.Allocatable[v1.ResourceMemory] memAllocatableMb = int((&mem).Value() / 1024 / 1024) - Expect(nodeCount).Should(Equal(sum)) + gomega.Expect(nodeCount).Should(gomega.Equal(sum)) if framework.ProviderIs("gke") { val, err := isAutoscalerEnabled(5) @@ -134,9 +134,9 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } }) - AfterEach(func() { + ginkgo.AfterEach(func() { framework.SkipUnlessProviderIs("gce", "gke") - By(fmt.Sprintf("Restoring initial size of the cluster")) + ginkgo.By(fmt.Sprintf("Restoring initial size of the cluster")) setMigSizes(originalSizes) expectedNodes := 0 for _, size := range originalSizes { @@ -163,29 +163,29 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { klog.Infof("Made nodes schedulable again in %v", time.Since(s).String()) }) - It("shouldn't increase cluster size if pending pod is too large [Feature:ClusterSizeAutoscalingScaleUp]", func() { - By("Creating unschedulable pod") + ginkgo.It("shouldn't increase cluster size if pending pod is too large [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.By("Creating unschedulable pod") ReserveMemory(f, "memory-reservation", 1, int(1.1*float64(memAllocatableMb)), false, defaultTimeout) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") - By("Waiting for scale up hoping it won't happen") + ginkgo.By("Waiting for scale up hoping it won't happen") // Verify that the appropriate event was generated eventFound := false EventsLoop: for start := time.Now(); time.Since(start) < scaleUpTimeout; time.Sleep(20 * time.Second) { - By("Waiting for NotTriggerScaleUp event") + ginkgo.By("Waiting for NotTriggerScaleUp event") events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(metav1.ListOptions{}) framework.ExpectNoError(err) for _, e := range events.Items { if e.InvolvedObject.Kind == "Pod" && e.Reason == "NotTriggerScaleUp" && strings.Contains(e.Message, "it wouldn't fit if a new node is added") { - By("NotTriggerScaleUp event found") + ginkgo.By("NotTriggerScaleUp event found") eventFound = true break EventsLoop } } } - Expect(eventFound).Should(Equal(true)) + gomega.Expect(eventFound).Should(gomega.Equal(true)) // Verify that cluster size is not changed framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size <= nodeCount }, time.Second)) @@ -201,12 +201,12 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) } - It("should increase cluster size if pending pods are small [Feature:ClusterSizeAutoscalingScaleUp]", + ginkgo.It("should increase cluster size if pending pods are small [Feature:ClusterSizeAutoscalingScaleUp]", func() { simpleScaleUpTest(0) }) gpuType := os.Getenv("TESTED_GPU_TYPE") - It(fmt.Sprintf("Should scale up GPU pool from 0 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { + ginkgo.It(fmt.Sprintf("Should scale up GPU pool from 0 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { framework.SkipUnlessProviderIs("gke") if gpuType == "" { framework.Failf("TEST_GPU_TYPE not defined") @@ -219,21 +219,21 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { installNvidiaDriversDaemonSet() - By("Enable autoscaler") + ginkgo.By("Enable autoscaler") framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) defer disableAutoscaler(gpuPoolName, 0, 1) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(0)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(0)) - By("Schedule a pod which requires GPU") + ginkgo.By("Schedule a pod which requires GPU") framework.ExpectNoError(ScheduleAnySingleGpuPod(f, "gpu-pod-rc")) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "gpu-pod-rc") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == nodeCount+1 }, scaleUpTimeout)) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(1)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(1)) }) - It(fmt.Sprintf("Should scale up GPU pool from 1 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { + ginkgo.It(fmt.Sprintf("Should scale up GPU pool from 1 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { framework.SkipUnlessProviderIs("gke") if gpuType == "" { framework.Failf("TEST_GPU_TYPE not defined") @@ -246,24 +246,24 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { installNvidiaDriversDaemonSet() - By("Schedule a single pod which requires GPU") + ginkgo.By("Schedule a single pod which requires GPU") framework.ExpectNoError(ScheduleAnySingleGpuPod(f, "gpu-pod-rc")) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - By("Enable autoscaler") + ginkgo.By("Enable autoscaler") framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 2)) defer disableAutoscaler(gpuPoolName, 0, 2) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(1)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(1)) - By("Scale GPU deployment") + ginkgo.By("Scale GPU deployment") framework.ScaleRC(f.ClientSet, f.ScalesGetter, f.Namespace.Name, "gpu-pod-rc", 2, true) framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == nodeCount+2 }, scaleUpTimeout)) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(2)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(2)) }) - It(fmt.Sprintf("Should not scale GPU pool up if pod does not require GPUs [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { + ginkgo.It(fmt.Sprintf("Should not scale GPU pool up if pod does not require GPUs [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { framework.SkipUnlessProviderIs("gke") if gpuType == "" { framework.Failf("TEST_GPU_TYPE not defined") @@ -276,12 +276,12 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { installNvidiaDriversDaemonSet() - By("Enable autoscaler") + ginkgo.By("Enable autoscaler") framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) defer disableAutoscaler(gpuPoolName, 0, 1) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(0)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(0)) - By("Schedule bunch of pods beyond point of filling default pool but do not request any GPUs") + ginkgo.By("Schedule bunch of pods beyond point of filling default pool but do not request any GPUs") ReserveMemory(f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") // Verify that cluster size is increased @@ -289,10 +289,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout)) // Expect gpu pool to stay intact - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(0)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(0)) }) - It(fmt.Sprintf("Should scale down GPU pool from 1 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { + ginkgo.It(fmt.Sprintf("Should scale down GPU pool from 1 [GpuType:%s] [Feature:ClusterSizeAutoscalingGpu]", gpuType), func() { framework.SkipUnlessProviderIs("gke") if gpuType == "" { framework.Failf("TEST_GPU_TYPE not defined") @@ -305,29 +305,29 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { installNvidiaDriversDaemonSet() - By("Schedule a single pod which requires GPU") + ginkgo.By("Schedule a single pod which requires GPU") framework.ExpectNoError(ScheduleAnySingleGpuPod(f, "gpu-pod-rc")) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "gpu-pod-rc") - By("Enable autoscaler") + ginkgo.By("Enable autoscaler") framework.ExpectNoError(enableAutoscaler(gpuPoolName, 0, 1)) defer disableAutoscaler(gpuPoolName, 0, 1) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(1)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(1)) - By("Remove the only POD requiring GPU") + ginkgo.By("Remove the only POD requiring GPU") framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "gpu-pod-rc") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == nodeCount }, scaleDownTimeout)) - Expect(len(getPoolNodes(f, gpuPoolName))).Should(Equal(0)) + gomega.Expect(len(getPoolNodes(f, gpuPoolName))).Should(gomega.Equal(0)) }) - It("should increase cluster size if pending pods are small and one node is broken [Feature:ClusterSizeAutoscalingScaleUp]", + ginkgo.It("should increase cluster size if pending pods are small and one node is broken [Feature:ClusterSizeAutoscalingScaleUp]", func() { framework.TestUnderTemporaryNetworkFailure(c, "default", getAnyNode(c), func() { simpleScaleUpTest(1) }) }) - It("shouldn't trigger additional scale-ups during processing scale-up [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("shouldn't trigger additional scale-ups during processing scale-up [Feature:ClusterSizeAutoscalingScaleUp]", func() { // Wait for the situation to stabilize - CA should be running and have up-to-date node readiness info. status, err := waitForScaleUpStatus(c, func(s *scaleUpStatus) bool { return s.ready == s.target && s.ready <= nodeCount @@ -336,7 +336,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { unmanagedNodes := nodeCount - status.ready - By("Schedule more pods than can fit and wait for cluster to scale-up") + ginkgo.By("Schedule more pods than can fit and wait for cluster to scale-up") ReserveMemory(f, "memory-reservation", 100, nodeCount*memAllocatableMb, false, 1*time.Second) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") @@ -347,7 +347,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { target := status.target framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) - By("Expect no more scale-up to be happening after all pods are scheduled") + ginkgo.By("Expect no more scale-up to be happening after all pods are scheduled") // wait for a while until scale-up finishes; we cannot read CA status immediately // after pods are scheduled as status config map is updated by CA once every loop iteration @@ -359,16 +359,16 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { if status.target != target { klog.Warningf("Final number of nodes (%v) does not match initial scale-up target (%v).", status.target, target) } - Expect(status.timestamp.Add(freshStatusLimit).Before(time.Now())).Should(Equal(false)) - Expect(status.status).Should(Equal(caNoScaleUpStatus)) - Expect(status.ready).Should(Equal(status.target)) - Expect(len(framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items)).Should(Equal(status.target + unmanagedNodes)) + gomega.Expect(status.timestamp.Add(freshStatusLimit).Before(time.Now())).Should(gomega.Equal(false)) + gomega.Expect(status.status).Should(gomega.Equal(caNoScaleUpStatus)) + gomega.Expect(status.ready).Should(gomega.Equal(status.target)) + gomega.Expect(len(framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items)).Should(gomega.Equal(status.target + unmanagedNodes)) }) - It("should increase cluster size if pending pods are small and there is another node pool that is not autoscaled [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should increase cluster size if pending pods are small and there is another node pool that is not autoscaled [Feature:ClusterSizeAutoscalingScaleUp]", func() { framework.SkipUnlessProviderIs("gke") - By("Creating new node-pool with n1-standard-4 machines") + ginkgo.By("Creating new node-pool with n1-standard-4 machines") const extraPoolName = "extra-pool" addNodePool(extraPoolName, "n1-standard-4", 1) defer deleteNodePool(extraPoolName) @@ -379,16 +379,16 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, resizeTimeout)) klog.Infof("Not enabling cluster autoscaler for the node pool (on purpose).") - By("Getting memory available on new nodes, so we can account for it when creating RC") + ginkgo.By("Getting memory available on new nodes, so we can account for it when creating RC") nodes := getPoolNodes(f, extraPoolName) - Expect(len(nodes)).Should(Equal(extraNodes)) + gomega.Expect(len(nodes)).Should(gomega.Equal(extraNodes)) extraMemMb := 0 for _, node := range nodes { mem := node.Status.Allocatable[v1.ResourceMemory] extraMemMb += int((&mem).Value() / 1024 / 1024) } - By("Reserving 0.1x more memory than the cluster holds to trigger scale up") + ginkgo.By("Reserving 0.1x more memory than the cluster holds to trigger scale up") totalMemoryReservation := int(1.1 * float64(nodeCount*memAllocatableMb+extraMemMb)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") ReserveMemory(f, "memory-reservation", 100, totalMemoryReservation, false, defaultTimeout) @@ -399,10 +399,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) }) - It("should disable node pool autoscaling [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should disable node pool autoscaling [Feature:ClusterSizeAutoscalingScaleUp]", func() { framework.SkipUnlessProviderIs("gke") - By("Creating new node-pool with n1-standard-4 machines") + ginkgo.By("Creating new node-pool with n1-standard-4 machines") const extraPoolName = "extra-pool" addNodePool(extraPoolName, "n1-standard-4", 1) defer deleteNodePool(extraPoolName) @@ -412,7 +412,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(disableAutoscaler(extraPoolName, 1, 2)) }) - It("should increase cluster size if pods are pending due to host port conflict [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should increase cluster size if pods are pending due to host port conflict [Feature:ClusterSizeAutoscalingScaleUp]", func() { scheduling.CreateHostPortPods(f, "host-port", nodeCount+2, false) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "host-port") @@ -421,18 +421,18 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) }) - It("should increase cluster size if pods are pending due to pod anti-affinity [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should increase cluster size if pods are pending due to pod anti-affinity [Feature:ClusterSizeAutoscalingScaleUp]", func() { pods := nodeCount newPods := 2 labels := map[string]string{ "anti-affinity": "yes", } - By("starting a pod with anti-affinity on each node") + ginkgo.By("starting a pod with anti-affinity on each node") framework.ExpectNoError(runAntiAffinityPods(f, f.Namespace.Name, pods, "some-pod", labels, labels)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "some-pod") framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) - By("scheduling extra pods with anti-affinity to existing ones") + ginkgo.By("scheduling extra pods with anti-affinity to existing ones") framework.ExpectNoError(runAntiAffinityPods(f, f.Namespace.Name, newPods, "extra-pod", labels, labels)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "extra-pod") @@ -440,8 +440,8 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount+newPods, scaleUpTimeout)) }) - It("should increase cluster size if pod requesting EmptyDir volume is pending [Feature:ClusterSizeAutoscalingScaleUp]", func() { - By("creating pods") + ginkgo.It("should increase cluster size if pod requesting EmptyDir volume is pending [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.By("creating pods") pods := nodeCount newPods := 1 labels := map[string]string{ @@ -450,10 +450,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(runAntiAffinityPods(f, f.Namespace.Name, pods, "some-pod", labels, labels)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "some-pod") - By("waiting for all pods before triggering scale up") + ginkgo.By("waiting for all pods before triggering scale up") framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) - By("creating a pod requesting EmptyDir") + ginkgo.By("creating a pod requesting EmptyDir") framework.ExpectNoError(runVolumeAntiAffinityPods(f, f.Namespace.Name, newPods, "extra-pod", labels, labels, emptyDirVolumes)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "extra-pod") @@ -461,7 +461,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount+newPods, scaleUpTimeout)) }) - It("should increase cluster size if pod requesting volume is pending [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should increase cluster size if pod requesting volume is pending [Feature:ClusterSizeAutoscalingScaleUp]", func() { framework.SkipUnlessProviderIs("gce", "gke") volumeLabels := labels.Set{ @@ -469,7 +469,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } selector := metav1.SetAsLabelSelector(volumeLabels) - By("creating volume & pvc") + ginkgo.By("creating volume & pvc") diskName, err := framework.CreatePDWithRetry() framework.ExpectNoError(err) pvConfig := framework.PersistentVolumeConfig{ @@ -505,7 +505,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } }() - By("creating pods") + ginkgo.By("creating pods") pods := nodeCount labels := map[string]string{ "anti-affinity": "yes", @@ -516,10 +516,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { klog.Infof("RC and pods not using volume deleted") }() - By("waiting for all pods before triggering scale up") + ginkgo.By("waiting for all pods before triggering scale up") framework.ExpectNoError(waitForAllCaPodsReadyInNamespace(f, c)) - By("creating a pod requesting PVC") + ginkgo.By("creating a pod requesting PVC") pvcPodName := "pvc-pod" newPods := 1 volumes := buildVolumes(pv, pvc) @@ -533,11 +533,11 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount+newPods, scaleUpTimeout)) }) - It("should add node to the particular mig [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should add node to the particular mig [Feature:ClusterSizeAutoscalingScaleUp]", func() { labelKey := "cluster-autoscaling-test.special-node" labelValue := "true" - By("Finding the smallest MIG") + ginkgo.By("Finding the smallest MIG") minMig := "" minSize := nodeCount for mig, size := range originalSizes { @@ -557,7 +557,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } removeLabels := func(nodesToClean sets.String) { - By("Removing labels from nodes") + ginkgo.By("Removing labels from nodes") for node := range nodesToClean { framework.RemoveLabelOffNode(c, node, labelKey) } @@ -567,7 +567,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(err) nodesSet := sets.NewString(nodes...) defer removeLabels(nodesSet) - By(fmt.Sprintf("Annotating nodes of the smallest MIG(%s): %v", minMig, nodes)) + ginkgo.By(fmt.Sprintf("Annotating nodes of the smallest MIG(%s): %v", minMig, nodes)) for node := range nodesSet { framework.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) @@ -575,7 +575,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { scheduling.CreateNodeSelectorPods(f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false) - By("Waiting for new node to appear and annotating it") + ginkgo.By("Waiting for new node to appear and annotating it") framework.WaitForGroupSize(minMig, int32(minSize+1)) // Verify that cluster size is increased framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, @@ -586,7 +586,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { newNodesSet := sets.NewString(newNodes...) newNodesSet.Delete(nodes...) if len(newNodesSet) > 1 { - By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet)) + ginkgo.By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet)) klog.Infof("Usually only 1 new node is expected, investigating") klog.Infof("Kubectl:%s\n", framework.RunKubectlOrDie("get", "nodes", "-o", "json")) if output, err := exec.Command("gcloud", "compute", "instances", "list", @@ -612,7 +612,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // However at this moment we DO WANT it to crash so that we don't check all test runs for the // rare behavior, but only the broken ones. } - By(fmt.Sprintf("New nodes: %v\n", newNodesSet)) + ginkgo.By(fmt.Sprintf("New nodes: %v\n", newNodesSet)) registeredNodes := sets.NewString() for nodeName := range newNodesSet { node, err := f.ClientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) @@ -622,7 +622,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { klog.Errorf("Failed to get node %v: %v", nodeName, err) } } - By(fmt.Sprintf("Setting labels for registered new nodes: %v", registeredNodes.List())) + ginkgo.By(fmt.Sprintf("Setting labels for registered new nodes: %v", registeredNodes.List())) for node := range registeredNodes { framework.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue) } @@ -633,10 +633,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "node-selector")) }) - It("should scale up correct target pool [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should scale up correct target pool [Feature:ClusterSizeAutoscalingScaleUp]", func() { framework.SkipUnlessProviderIs("gke") - By("Creating new node-pool with n1-standard-4 machines") + ginkgo.By("Creating new node-pool with n1-standard-4 machines") const extraPoolName = "extra-pool" addNodePool(extraPoolName, "n1-standard-4", 1) defer deleteNodePool(extraPoolName) @@ -647,7 +647,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { extraPods := extraNodes + 1 totalMemoryReservation := int(float64(extraPods) * 1.5 * float64(memAllocatableMb)) - By(fmt.Sprintf("Creating rc with %v pods too big to fit default-pool but fitting extra-pool", extraPods)) + ginkgo.By(fmt.Sprintf("Creating rc with %v pods too big to fit default-pool but fitting extra-pool", extraPods)) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") ReserveMemory(f, "memory-reservation", extraPods, totalMemoryReservation, false, defaultTimeout) @@ -663,7 +663,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { defer cleanup() framework.ExpectNoError(err) - By("Manually increase cluster size") + ginkgo.By("Manually increase cluster size") increasedSize := 0 newSizes := make(map[string]int) for key, val := range originalSizes { @@ -674,20 +674,20 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(f.ClientSet, func(size int) bool { return size >= increasedSize }, manualResizeTimeout, unready)) - By("Some node should be removed") + ginkgo.By("Some node should be removed") framework.ExpectNoError(WaitForClusterSizeFuncWithUnready(f.ClientSet, func(size int) bool { return size < increasedSize }, scaleDownTimeout, unready)) } - It("should correctly scale down after a node is not needed [Feature:ClusterSizeAutoscalingScaleDown]", + ginkgo.It("should correctly scale down after a node is not needed [Feature:ClusterSizeAutoscalingScaleDown]", func() { simpleScaleDownTest(0) }) - It("should correctly scale down after a node is not needed and one node is broken [Feature:ClusterSizeAutoscalingScaleDown]", + ginkgo.It("should correctly scale down after a node is not needed and one node is broken [Feature:ClusterSizeAutoscalingScaleDown]", func() { framework.TestUnderTemporaryNetworkFailure(c, "default", getAnyNode(c), func() { simpleScaleDownTest(1) }) }) - It("should correctly scale down after a node is not needed when there is non autoscaled pool[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("should correctly scale down after a node is not needed when there is non autoscaled pool[Feature:ClusterSizeAutoscalingScaleDown]", func() { framework.SkipUnlessProviderIs("gke") increasedSize := manuallyIncreaseClusterSize(f, originalSizes) @@ -700,7 +700,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size >= increasedSize+extraNodes }, scaleUpTimeout)) - By("Some node should be removed") + ginkgo.By("Some node should be removed") // Apparently GKE master is restarted couple minutes after the node pool is added // reseting all the timers in scale down code. Adding 10 extra minutes to workaround // this issue. @@ -709,44 +709,44 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { func(size int) bool { return size < increasedSize+extraNodes }, scaleDownTimeout+10*time.Minute)) }) - It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() { runDrainTest(f, originalSizes, f.Namespace.Name, 1, 1, func(increasedSize int) { - By("Some node should be removed") + ginkgo.By("Some node should be removed") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size < increasedSize }, scaleDownTimeout)) }) }) - It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() { runDrainTest(f, originalSizes, f.Namespace.Name, 1, 0, func(increasedSize int) { - By("No nodes should be removed") + ginkgo.By("No nodes should be removed") time.Sleep(scaleDownTimeout) nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) - Expect(len(nodes.Items)).Should(Equal(increasedSize)) + gomega.Expect(len(nodes.Items)).Should(gomega.Equal(increasedSize)) }) }) - It("should be able to scale down by draining multiple pods one by one as dictated by pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("should be able to scale down by draining multiple pods one by one as dictated by pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() { runDrainTest(f, originalSizes, f.Namespace.Name, 2, 1, func(increasedSize int) { - By("Some node should be removed") + ginkgo.By("Some node should be removed") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size < increasedSize }, scaleDownTimeout)) }) }) - It("should be able to scale down by draining system pods with pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("should be able to scale down by draining system pods with pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() { runDrainTest(f, originalSizes, "kube-system", 2, 1, func(increasedSize int) { - By("Some node should be removed") + ginkgo.By("Some node should be removed") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size < increasedSize }, scaleDownTimeout)) }) }) - It("Should be able to scale a node group up from 0[Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("Should be able to scale a node group up from 0[Feature:ClusterSizeAutoscalingScaleUp]", func() { // Provider-specific setup if framework.ProviderIs("gke") { // GKE-specific setup - By("Add a new node pool with 0 nodes and min size 0") + ginkgo.By("Add a new node pool with 0 nodes and min size 0") const extraPoolName = "extra-pool" addNodePool(extraPoolName, "n1-standard-4", 0) defer deleteNodePool(extraPoolName) @@ -756,7 +756,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // on GCE, run only if there are already at least 2 node groups framework.SkipUnlessAtLeast(len(originalSizes), 2, "At least 2 node groups are needed for scale-to-0 tests") - By("Manually scale smallest node group to 0") + ginkgo.By("Manually scale smallest node group to 0") minMig := "" minSize := nodeCount for mig, size := range originalSizes { @@ -769,7 +769,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount-minSize, resizeTimeout)) } - By("Make remaining nodes unschedulable") + ginkgo.By("Make remaining nodes unschedulable") nodes, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ "spec.unschedulable": "false", }.AsSelector().String()}) @@ -785,7 +785,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(err) } - By("Run a scale-up test") + ginkgo.By("Run a scale-up test") ReserveMemory(f, "memory-reservation", 1, 100, false, 1*time.Second) defer framework.DeleteRCAndWaitForGC(f.ClientSet, f.Namespace.Name, "memory-reservation") @@ -807,7 +807,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // verify the targeted node pool/MIG is of size 0 gkeScaleToZero := func() { // GKE-specific setup - By("Add a new node pool with size 1 and min size 0") + ginkgo.By("Add a new node pool with size 1 and min size 0") const extraPoolName = "extra-pool" addNodePool(extraPoolName, "n1-standard-4", 1) defer deleteNodePool(extraPoolName) @@ -817,9 +817,9 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { defer disableAutoscaler(extraPoolName, 0, 1) ngNodes := getPoolNodes(f, extraPoolName) - Expect(len(ngNodes)).To(Equal(extraNodes)) + gomega.Expect(len(ngNodes)).To(gomega.Equal(extraNodes)) for _, node := range ngNodes { - By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) + ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) } for _, node := range ngNodes { @@ -830,12 +830,12 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // GKE-specific check newSize := getPoolSize(f, extraPoolName) - Expect(newSize).Should(Equal(0)) + gomega.Expect(newSize).Should(gomega.Equal(0)) } gceScaleToZero := func() { // non-GKE only - By("Find smallest node group and manually scale it to a single node") + ginkgo.By("Find smallest node group and manually scale it to a single node") minMig := "" minSize := nodeCount for mig, size := range originalSizes { @@ -848,9 +848,9 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount-minSize+1, resizeTimeout)) ngNodes, err := framework.GetGroupNodes(minMig) framework.ExpectNoError(err) - Expect(len(ngNodes) == 1).To(BeTrue()) + gomega.Expect(len(ngNodes) == 1).To(gomega.BeTrue()) node, err := f.ClientSet.CoreV1().Nodes().Get(ngNodes[0], metav1.GetOptions{}) - By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) + ginkgo.By(fmt.Sprintf("Target node for scale-down: %s", node.Name)) framework.ExpectNoError(err) // this part is identical @@ -861,10 +861,10 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // non-GKE only newSize, err := framework.GroupSize(minMig) framework.ExpectNoError(err) - Expect(newSize).Should(Equal(0)) + gomega.Expect(newSize).Should(gomega.Equal(0)) } - It("Should be able to scale a node group down to 0[Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("Should be able to scale a node group down to 0[Feature:ClusterSizeAutoscalingScaleDown]", func() { if framework.ProviderIs("gke") { // In GKE, we can just add a node pool gkeScaleToZero() } else if len(originalSizes) >= 2 { @@ -874,7 +874,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { } }) - It("Shouldn't perform scale up operation and should list unhealthy status if most of the cluster is broken[Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("Shouldn't perform scale up operation and should list unhealthy status if most of the cluster is broken[Feature:ClusterSizeAutoscalingScaleUp]", func() { clusterSize := nodeCount for clusterSize < unhealthyClusterThreshold+1 { clusterSize = manuallyIncreaseClusterSize(f, originalSizes) @@ -893,13 +893,13 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { // making no assumptions about minimal node startup time. time.Sleep(2 * time.Minute) - By("Block network connectivity to some nodes to simulate unhealthy cluster") + ginkgo.By("Block network connectivity to some nodes to simulate unhealthy cluster") nodesToBreakCount := int(math.Ceil(math.Max(float64(unhealthyClusterThreshold), 0.5*float64(clusterSize)))) nodes, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ "spec.unschedulable": "false", }.AsSelector().String()}) framework.ExpectNoError(err) - Expect(nodesToBreakCount <= len(nodes.Items)).To(BeTrue()) + gomega.Expect(nodesToBreakCount <= len(nodes.Items)).To(gomega.BeTrue()) nodesToBreak := nodes.Items[:nodesToBreakCount] // TestUnderTemporaryNetworkFailure only removes connectivity to a single node, @@ -917,11 +917,11 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { time.Sleep(scaleUpTimeout) currentNodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) e2elog.Logf("Currently available nodes: %v, nodes available at the start of test: %v, disabled nodes: %v", len(currentNodes.Items), len(nodes.Items), nodesToBreakCount) - Expect(len(currentNodes.Items)).Should(Equal(len(nodes.Items) - nodesToBreakCount)) + gomega.Expect(len(currentNodes.Items)).Should(gomega.Equal(len(nodes.Items) - nodesToBreakCount)) status, err := getClusterwideStatus(c) e2elog.Logf("Clusterwide status: %v", status) framework.ExpectNoError(err) - Expect(status).Should(Equal("Unhealthy")) + gomega.Expect(status).Should(gomega.Equal("Unhealthy")) } } testFunction() @@ -929,19 +929,19 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { framework.ExpectNoError(framework.WaitForReadyNodes(c, len(nodes.Items), nodesRecoverTimeout)) }) - It("shouldn't scale up when expendable pod is created [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("shouldn't scale up when expendable pod is created [Feature:ClusterSizeAutoscalingScaleUp]", func() { defer createPriorityClasses(f)() // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. cleanupFunc := ReserveMemoryWithPriority(f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), false, time.Second, expendablePriorityClassName) defer cleanupFunc() - By(fmt.Sprintf("Waiting for scale up hoping it won't happen, sleep for %s", scaleUpTimeout.String())) + ginkgo.By(fmt.Sprintf("Waiting for scale up hoping it won't happen, sleep for %s", scaleUpTimeout.String())) time.Sleep(scaleUpTimeout) // Verify that cluster size is not changed framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == nodeCount }, time.Second)) }) - It("should scale up when non expendable pod is created [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("should scale up when non expendable pod is created [Feature:ClusterSizeAutoscalingScaleUp]", func() { defer createPriorityClasses(f)() // Create nodesCountAfterResize+1 pods allocating 0.7 allocatable on present nodes. One more node will have to be created. cleanupFunc := ReserveMemoryWithPriority(f, "memory-reservation", nodeCount+1, int(float64(nodeCount+1)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) @@ -951,7 +951,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { func(size int) bool { return size > nodeCount }, time.Second)) }) - It("shouldn't scale up when expendable pod is preempted [Feature:ClusterSizeAutoscalingScaleUp]", func() { + ginkgo.It("shouldn't scale up when expendable pod is preempted [Feature:ClusterSizeAutoscalingScaleUp]", func() { defer createPriorityClasses(f)() // Create nodesCountAfterResize pods allocating 0.7 allocatable on present nodes - one pod per node. cleanupFunc1 := ReserveMemoryWithPriority(f, "memory-reservation1", nodeCount, int(float64(nodeCount)*float64(0.7)*float64(memAllocatableMb)), true, defaultTimeout, expendablePriorityClassName) @@ -963,24 +963,24 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { func(size int) bool { return size == nodeCount }, time.Second)) }) - It("should scale down when expendable pod is running [Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("should scale down when expendable pod is running [Feature:ClusterSizeAutoscalingScaleDown]", func() { defer createPriorityClasses(f)() increasedSize := manuallyIncreaseClusterSize(f, originalSizes) // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. cleanupFunc := ReserveMemoryWithPriority(f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, expendablePriorityClassName) defer cleanupFunc() - By("Waiting for scale down") + ginkgo.By("Waiting for scale down") framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == nodeCount }, scaleDownTimeout)) }) - It("shouldn't scale down when non expendable pod is running [Feature:ClusterSizeAutoscalingScaleDown]", func() { + ginkgo.It("shouldn't scale down when non expendable pod is running [Feature:ClusterSizeAutoscalingScaleDown]", func() { defer createPriorityClasses(f)() increasedSize := manuallyIncreaseClusterSize(f, originalSizes) // Create increasedSize pods allocating 0.7 allocatable on present nodes - one pod per node. cleanupFunc := ReserveMemoryWithPriority(f, "memory-reservation", increasedSize, int(float64(increasedSize)*float64(0.7)*float64(memAllocatableMb)), true, scaleUpTimeout, highPriorityClassName) defer cleanupFunc() - By(fmt.Sprintf("Waiting for scale down hoping it won't happen, sleep for %s", scaleDownTimeout.String())) + ginkgo.By(fmt.Sprintf("Waiting for scale down hoping it won't happen, sleep for %s", scaleDownTimeout.String())) time.Sleep(scaleDownTimeout) framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, func(size int) bool { return size == increasedSize }, time.Second)) @@ -988,7 +988,7 @@ var _ = SIGDescribe("Cluster size autoscaling [Slow]", func() { }) func installNvidiaDriversDaemonSet() { - By("Add daemonset which installs nvidia drivers") + ginkgo.By("Add daemonset which installs nvidia drivers") // the link differs from one in GKE documentation; discussed with @mindprince this one should be used framework.RunKubectlOrDie("apply", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml") } @@ -1012,7 +1012,7 @@ func runDrainTest(f *framework.Framework, migSizes map[string]int, namespace str defer framework.DeleteRCAndWaitForGC(f.ClientSet, namespace, "reschedulable-pods") - By("Create a PodDisruptionBudget") + ginkgo.By("Create a PodDisruptionBudget") minAvailable := intstr.FromInt(numPods - pdbSize) pdb := &policy.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ @@ -1034,15 +1034,15 @@ func runDrainTest(f *framework.Framework, migSizes map[string]int, namespace str verifyFunction(increasedSize) } -func getGkeApiEndpoint() string { - gkeApiEndpoint := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER") - if gkeApiEndpoint == "" { - gkeApiEndpoint = "https://test-container.sandbox.googleapis.com" +func getGkeAPIEndpoint() string { + gkeAPIEndpoint := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER") + if gkeAPIEndpoint == "" { + gkeAPIEndpoint = "https://test-container.sandbox.googleapis.com" } - if strings.HasSuffix(gkeApiEndpoint, "/") { - gkeApiEndpoint = gkeApiEndpoint[:len(gkeApiEndpoint)-1] + if strings.HasSuffix(gkeAPIEndpoint, "/") { + gkeAPIEndpoint = gkeAPIEndpoint[:len(gkeAPIEndpoint)-1] } - return gkeApiEndpoint + return gkeAPIEndpoint } func getGKEURL(apiVersion string, suffix string) string { @@ -1051,7 +1051,7 @@ func getGKEURL(apiVersion string, suffix string) string { token := strings.Replace(string(out), "\n", "", -1) return fmt.Sprintf("%s/%s/%s?access_token=%s", - getGkeApiEndpoint(), + getGkeAPIEndpoint(), apiVersion, suffix, token) @@ -1064,12 +1064,11 @@ func getGKEClusterURL(apiVersion string) string { framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Cluster)) - } else { - return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/zones/%s/clusters/%s", - framework.TestContext.CloudConfig.ProjectID, - framework.TestContext.CloudConfig.Zone, - framework.TestContext.CloudConfig.Cluster)) } + return getGKEURL(apiVersion, fmt.Sprintf("projects/%s/zones/%s/clusters/%s", + framework.TestContext.CloudConfig.ProjectID, + framework.TestContext.CloudConfig.Zone, + framework.TestContext.CloudConfig.Cluster)) } func getCluster(apiVersion string) (string, error) { @@ -1107,9 +1106,8 @@ func isAutoscalerEnabled(expectedMaxNodeCountInTargetPool int) (bool, error) { func getClusterLocation() string { if isRegionalCluster() { return "--region=" + framework.TestContext.CloudConfig.Region - } else { - return "--zone=" + framework.TestContext.CloudConfig.Zone } + return "--zone=" + framework.TestContext.CloudConfig.Zone } func getGcloudCommandFromTrack(commandTrack string, args []string) []string { @@ -1248,7 +1246,7 @@ func getPoolInitialSize(poolName string) int { klog.Infof("Node-pool initial size: %s", output) framework.ExpectNoError(err, string(output)) fields := strings.Fields(string(output)) - Expect(len(fields)).Should(Equal(1)) + gomega.Expect(len(fields)).Should(gomega.Equal(1)) size, err := strconv.ParseInt(fields[0], 10, 64) framework.ExpectNoError(err) @@ -1274,7 +1272,7 @@ func getPoolSize(f *framework.Framework, poolName string) int { } func reserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration, priorityClassName string) func() error { - By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) + ginkgo.By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) request := int64(1024 * 1024 * megabytes / replicas) config := &testutils.RCConfig{ Client: f.ClientSet, @@ -1311,7 +1309,7 @@ func ReserveMemoryWithPriority(f *framework.Framework, id string, replicas, mega return reserveMemory(f, id, replicas, megabytes, expectRunning, timeout, nil, nil, priorityClassName) } -// ReserveMemoryWithSelector creates a replication controller with pods with node selector that, in summation, +// ReserveMemoryWithSelectorAndTolerations creates a replication controller with pods with node selector that, in summation, // request the specified amount of memory. func ReserveMemoryWithSelectorAndTolerations(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration, selector map[string]string, tolerations []v1.Toleration) func() error { return reserveMemory(f, id, replicas, megabytes, expectRunning, timeout, selector, tolerations, "") @@ -1418,7 +1416,7 @@ func setMigSizes(sizes map[string]int) bool { currentSize, err := framework.GroupSize(mig) framework.ExpectNoError(err) if desiredSize != currentSize { - By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize)) + ginkgo.By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize)) err = framework.ResizeGroup(mig, int32(desiredSize)) framework.ExpectNoError(err) madeChanges = true @@ -1428,10 +1426,10 @@ func setMigSizes(sizes map[string]int) bool { } func drainNode(f *framework.Framework, node *v1.Node) { - By("Make the single node unschedulable") + ginkgo.By("Make the single node unschedulable") makeNodeUnschedulable(f.ClientSet, node) - By("Manually drain the single node") + ginkgo.By("Manually drain the single node") podOpts := metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, node.Name).String()} pods, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceAll).List(podOpts) framework.ExpectNoError(err) @@ -1442,7 +1440,7 @@ func drainNode(f *framework.Framework, node *v1.Node) { } func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error { - By(fmt.Sprintf("Taint node %s", node.Name)) + ginkgo.By(fmt.Sprintf("Taint node %s", node.Name)) for j := 0; j < 3; j++ { freshNode, err := c.CoreV1().Nodes().Get(node.Name, metav1.GetOptions{}) if err != nil { @@ -1479,7 +1477,7 @@ func (CriticalAddonsOnlyError) Error() string { } func makeNodeSchedulable(c clientset.Interface, node *v1.Node, failOnCriticalAddonsOnly bool) error { - By(fmt.Sprintf("Remove taint from node %s", node.Name)) + ginkgo.By(fmt.Sprintf("Remove taint from node %s", node.Name)) for j := 0; j < 3; j++ { freshNode, err := c.CoreV1().Nodes().Get(node.Name, metav1.GetOptions{}) if err != nil { @@ -1634,7 +1632,7 @@ func buildAntiAffinity(labels map[string]string) *v1.Affinity { // 3a. enable scheduling on that node // 3b. increase number of replicas in RC by podsPerNode func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, namespace string, podsPerNode int, id string, labels map[string]string, memRequest int64) error { - By("Run a pod on each node") + ginkgo.By("Run a pod on each node") for _, node := range nodes { err := makeNodeUnschedulable(f.ClientSet, &node) @@ -1709,7 +1707,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, namespa // Increase cluster size by newNodesForScaledownTests to create some unused nodes // that can be later removed by cluster autoscaler. func manuallyIncreaseClusterSize(f *framework.Framework, originalSizes map[string]int) int { - By("Manually increase cluster size") + ginkgo.By("Manually increase cluster size") increasedSize := 0 newSizes := make(map[string]int) for key, val := range originalSizes { @@ -1857,13 +1855,13 @@ func waitForScaleUpStatus(c clientset.Interface, cond func(s *scaleUpStatus) boo // This is a temporary fix to allow CA to migrate some kube-system pods // TODO: Remove this when the PDB is added for some of those components func addKubeSystemPdbs(f *framework.Framework) (func(), error) { - By("Create PodDisruptionBudgets for kube-system components, so they can be migrated if required") + ginkgo.By("Create PodDisruptionBudgets for kube-system components, so they can be migrated if required") var newPdbs []string cleanup := func() { var finalErr error for _, newPdbName := range newPdbs { - By(fmt.Sprintf("Delete PodDisruptionBudget %v", newPdbName)) + ginkgo.By(fmt.Sprintf("Delete PodDisruptionBudget %v", newPdbName)) err := f.ClientSet.PolicyV1beta1().PodDisruptionBudgets("kube-system").Delete(newPdbName, &metav1.DeleteOptions{}) if err != nil { // log error, but attempt to remove other pdbs @@ -1888,7 +1886,7 @@ func addKubeSystemPdbs(f *framework.Framework) (func(), error) { {label: "glbc", minAvailable: 0}, } for _, pdbData := range pdbsToAdd { - By(fmt.Sprintf("Create PodDisruptionBudget for %v", pdbData.label)) + ginkgo.By(fmt.Sprintf("Create PodDisruptionBudget for %v", pdbData.label)) labelMap := map[string]string{"k8s-app": pdbData.label} pdbName := fmt.Sprintf("test-pdb-for-%v", pdbData.label) minAvailable := intstr.FromInt(pdbData.minAvailable) @@ -1922,7 +1920,7 @@ func createPriorityClasses(f *framework.Framework) func() { if err != nil { klog.Errorf("Error creating priority class: %v", err) } - Expect(err == nil || errors.IsAlreadyExists(err)).To(Equal(true)) + gomega.Expect(err == nil || errors.IsAlreadyExists(err)).To(gomega.Equal(true)) } return func() { diff --git a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go index 9556b6d5f09..1727b29c711 100644 --- a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go @@ -33,7 +33,7 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" - . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo" "golang.org/x/oauth2/google" ) @@ -45,13 +45,13 @@ const ( ) var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() { - BeforeEach(func() { + ginkgo.BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") }) f := framework.NewDefaultFramework("horizontal-pod-autoscaling") - It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 // metric should cause scale down metricValue := int64(100) @@ -66,7 +66,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 // metric should cause scale down metricValue := int64(100) @@ -83,7 +83,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale down with External Metric with target value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale down with External Metric with target value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 // metric should cause scale down metricValue := externalMetricValue @@ -106,7 +106,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale down with External Metric with target average value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale down with External Metric with target average value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 // metric should cause scale down metricValue := externalMetricValue @@ -129,7 +129,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 // metric should cause scale down metricValue := int64(100) @@ -144,7 +144,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 1 // metric 1 would cause a scale down, if not for metric 2 metric1Value := int64(100) @@ -175,7 +175,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me tc.Run() }) - It("should scale up with two External metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + ginkgo.It("should scale up with two External metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 1 // metric 1 would cause a scale down, if not for metric 2 metric1Value := externalMetricValue @@ -216,6 +216,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me }) }) +// CustomMetricTestCase is a struct for test cases. type CustomMetricTestCase struct { framework *framework.Framework hpa *as.HorizontalPodAutoscaler @@ -226,8 +227,9 @@ type CustomMetricTestCase struct { scaledReplicas int } +// Run starts test case. func (tc *CustomMetricTestCase) Run() { - projectId := framework.TestContext.CloudConfig.ProjectID + projectID := framework.TestContext.CloudConfig.ProjectID ctx := context.Background() client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) @@ -251,11 +253,11 @@ func (tc *CustomMetricTestCase) Run() { } // Set up a cluster: create a custom metric and set up k8s-sd adapter - err = monitoring.CreateDescriptors(gcmService, projectId) + err = monitoring.CreateDescriptors(gcmService, projectID) if err != nil { framework.Failf("Failed to create metric descriptor: %v", err) } - defer monitoring.CleanupDescriptors(gcmService, projectId) + defer monitoring.CleanupDescriptors(gcmService, projectID) err = monitoring.CreateAdapter(monitoring.AdapterDefault) if err != nil { diff --git a/test/e2e/autoscaling/dns_autoscaling.go b/test/e2e/autoscaling/dns_autoscaling.go index a2bad195cbd..97113b22eba 100644 --- a/test/e2e/autoscaling/dns_autoscaling.go +++ b/test/e2e/autoscaling/dns_autoscaling.go @@ -31,10 +31,11 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" ) +// Constants used in dns-autoscaling test. const ( DNSdefaultTimeout = 5 * time.Minute ClusterAddonLabelKey = "k8s-app" @@ -47,18 +48,18 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { var c clientset.Interface var previousParams map[string]string var originDNSReplicasCount int - var DNSParams_1 DNSParamsLinear - var DNSParams_2 DNSParamsLinear - var DNSParams_3 DNSParamsLinear + var DNSParams1 DNSParamsLinear + var DNSParams2 DNSParamsLinear + var DNSParams3 DNSParamsLinear - BeforeEach(func() { + ginkgo.BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") c = f.ClientSet nodeCount := len(framework.GetReadySchedulableNodesOrDie(c).Items) - Expect(nodeCount).NotTo(BeZero()) + gomega.Expect(nodeCount).NotTo(gomega.BeZero()) - By("Collecting original replicas count and DNS scaling params") + ginkgo.By("Collecting original replicas count and DNS scaling params") var err error originDNSReplicasCount, err = getDNSReplicas(c) framework.ExpectNoError(err) @@ -68,13 +69,13 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { previousParams = pcm.Data if nodeCount <= 500 { - DNSParams_1 = DNSParamsLinear{ + DNSParams1 = DNSParamsLinear{ nodesPerReplica: 1, } - DNSParams_2 = DNSParamsLinear{ + DNSParams2 = DNSParamsLinear{ nodesPerReplica: 2, } - DNSParams_3 = DNSParamsLinear{ + DNSParams3 = DNSParamsLinear{ nodesPerReplica: 3, coresPerReplica: 3, } @@ -84,13 +85,13 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { // The default setup is: 256 cores/replica, 16 nodes/replica. // With nodeCount > 500, nodes/13, nodes/14, nodes/15 and nodes/16 // are different numbers. - DNSParams_1 = DNSParamsLinear{ + DNSParams1 = DNSParamsLinear{ nodesPerReplica: 13, } - DNSParams_2 = DNSParamsLinear{ + DNSParams2 = DNSParamsLinear{ nodesPerReplica: 14, } - DNSParams_3 = DNSParamsLinear{ + DNSParams3 = DNSParamsLinear{ nodesPerReplica: 15, coresPerReplica: 15, } @@ -99,25 +100,25 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { // This test is separated because it is slow and need to run serially. // Will take around 5 minutes to run on a 4 nodes cluster. - It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { + ginkgo.It("[Serial] [Slow] kube-dns-autoscaler should scale kube-dns pods when cluster size changed", func() { numNodes, err := framework.NumberOfRegisteredNodes(c) framework.ExpectNoError(err) - By("Replace the dns autoscaling parameters with testing parameters") - err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + ginkgo.By("Replace the dns autoscaling parameters with testing parameters") + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams1))) framework.ExpectNoError(err) defer func() { - By("Restoring initial dns autoscaling parameters") + ginkgo.By("Restoring initial dns autoscaling parameters") err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams)) framework.ExpectNoError(err) - By("Wait for number of running and ready kube-dns pods recover") + ginkgo.By("Wait for number of running and ready kube-dns pods recover") label := labels.SelectorFromSet(labels.Set(map[string]string{ClusterAddonLabelKey: DNSLabelName})) _, err := framework.WaitForPodsWithLabelRunningReady(c, metav1.NamespaceSystem, label, originDNSReplicasCount, DNSdefaultTimeout) framework.ExpectNoError(err) }() - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams1) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) @@ -125,11 +126,11 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { size, err := framework.GroupSize(mig) framework.ExpectNoError(err) - By(fmt.Sprintf("Initial size of %s: %d", mig, size)) + ginkgo.By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size } - By("Manually increase cluster size") + ginkgo.By("Manually increase cluster size") increasedSizes := make(map[string]int) for key, val := range originalSizes { increasedSizes[key] = val + 1 @@ -139,87 +140,88 @@ var _ = SIGDescribe("DNS horizontal autoscaling", func() { func(size int) bool { return size == numNodes+len(originalSizes) }, scaleUpTimeout) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams1) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) - By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_3))) + ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams3))) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams3) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) - By("Restoring cluster size") + ginkgo.By("Restoring cluster size") setMigSizes(originalSizes) err = framework.WaitForReadyNodes(c, numNodes, scaleDownTimeout) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") + ginkgo.By("Wait for kube-dns scaled to expected number") err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) }) // TODO: Get rid of [DisabledForLargeClusters] tag when issue #55779 is fixed. - It("[DisabledForLargeClusters] kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { + ginkgo.It("[DisabledForLargeClusters] kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { - By("Replace the dns autoscaling parameters with testing parameters") - err := updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + ginkgo.By("Replace the dns autoscaling parameters with testing parameters") + err := updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams1))) framework.ExpectNoError(err) defer func() { - By("Restoring initial dns autoscaling parameters") + ginkgo.By("Restoring initial dns autoscaling parameters") err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams)) framework.ExpectNoError(err) }() - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams1) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) - By("--- Scenario: should scale kube-dns based on changed parameters ---") - By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_3))) + ginkgo.By("--- Scenario: should scale kube-dns based on changed parameters ---") + ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams3))) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams3) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) - By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") - By("Delete the ConfigMap for autoscaler") + ginkgo.By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") + ginkgo.By("Delete the ConfigMap for autoscaler") err = deleteDNSScalingConfigMap(c) framework.ExpectNoError(err) - By("Wait for the ConfigMap got re-created") + ginkgo.By("Wait for the ConfigMap got re-created") _, err = waitForDNSConfigMapCreated(c, DNSdefaultTimeout) framework.ExpectNoError(err) - By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_2))) + ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams2))) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_2) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams2) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) - By("--- Scenario: should recover after autoscaler pod got deleted ---") - By("Delete the autoscaler pod for kube-dns") + ginkgo.By("--- Scenario: should recover after autoscaler pod got deleted ---") + ginkgo.By("Delete the autoscaler pod for kube-dns") err = deleteDNSAutoscalerPod(c) framework.ExpectNoError(err) - By("Replace the dns autoscaling parameters with another testing parameters") - err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams_1))) + ginkgo.By("Replace the dns autoscaling parameters with another testing parameters") + err = updateDNSScalingConfigMap(c, packDNSScalingConfigMap(packLinearParams(&DNSParams1))) framework.ExpectNoError(err) - By("Wait for kube-dns scaled to expected number") - getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) + ginkgo.By("Wait for kube-dns scaled to expected number") + getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams1) err = waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout) framework.ExpectNoError(err) }) }) +// DNSParamsLinear is a struct for number of DNS pods. type DNSParamsLinear struct { nodesPerReplica float64 coresPerReplica float64 diff --git a/test/e2e/autoscaling/framework.go b/test/e2e/autoscaling/framework.go index 43b99e0f0de..6133b19cc6d 100644 --- a/test/e2e/autoscaling/framework.go +++ b/test/e2e/autoscaling/framework.go @@ -18,6 +18,7 @@ package autoscaling import "github.com/onsi/ginkgo" +// SIGDescribe annotates the test with the SIG label. func SIGDescribe(text string, body func()) bool { return ginkgo.Describe("[sig-autoscaling] "+text, body) } diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling.go b/test/e2e/autoscaling/horizontal_pod_autoscaling.go index 00737474245..1a5aed07085 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling.go @@ -23,7 +23,7 @@ import ( "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" - . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo" ) // These tests don't seem to be running properly in parallel: issue: #20338. @@ -37,20 +37,20 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: CPU)", fu SIGDescribe("[Serial] [Slow] Deployment", func() { // CPU tests via deployments - It(titleUp, func() { + ginkgo.It(titleUp, func() { scaleUp("test-deployment", common.KindDeployment, false, rc, f) }) - It(titleDown, func() { + ginkgo.It(titleDown, func() { scaleDown("test-deployment", common.KindDeployment, false, rc, f) }) }) SIGDescribe("[Serial] [Slow] ReplicaSet", func() { // CPU tests via ReplicaSets - It(titleUp, func() { + ginkgo.It(titleUp, func() { scaleUp("rs", common.KindReplicaSet, false, rc, f) }) - It(titleDown, func() { + ginkgo.It(titleDown, func() { scaleDown("rs", common.KindReplicaSet, false, rc, f) }) }) @@ -58,16 +58,16 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: CPU)", fu // These tests take ~20 minutes each. SIGDescribe("[Serial] [Slow] ReplicationController", func() { // CPU tests via replication controllers - It(titleUp+" and verify decision stability", func() { + ginkgo.It(titleUp+" and verify decision stability", func() { scaleUp("rc", common.KindRC, true, rc, f) }) - It(titleDown+" and verify decision stability", func() { + ginkgo.It(titleDown+" and verify decision stability", func() { scaleDown("rc", common.KindRC, true, rc, f) }) }) SIGDescribe("ReplicationController light", func() { - It("Should scale from 1 pod to 2 pods", func() { + ginkgo.It("Should scale from 1 pod to 2 pods", func() { scaleTest := &HPAScaleTest{ initPods: 1, totalInitialCPUUsage: 150, @@ -79,7 +79,7 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: CPU)", fu } scaleTest.run("rc-light", common.KindRC, rc, f) }) - It("Should scale from 2 pods to 1 pod", func() { + ginkgo.It("Should scale from 2 pods to 1 pod", func() { scaleTest := &HPAScaleTest{ initPods: 2, totalInitialCPUUsage: 50, diff --git a/test/e2e/common/configmap_volume.go b/test/e2e/common/configmap_volume.go index 985643e82b8..e518d5b3345 100644 --- a/test/e2e/common/configmap_volume.go +++ b/test/e2e/common/configmap_volume.go @@ -182,7 +182,7 @@ var _ = Describe("[sig-storage] ConfigMap", func() { configMap.ResourceVersion = "" // to force update configMap.Data["data-1"] = "value-2" _, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(configMap) - Expect(err).NotTo(HaveOccurred(), "Failed to update configmap %q in namespace %q", configMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update configmap %q in namespace %q", configMap.Name, f.Namespace.Name) By("waiting to observe update in volume") Eventually(pollLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-2")) @@ -446,14 +446,14 @@ var _ = Describe("[sig-storage] ConfigMap", func() { By(fmt.Sprintf("Deleting configmap %v", deleteConfigMap.Name)) err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(deleteConfigMap.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to delete configmap %q in namespace %q", deleteConfigMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to delete configmap %q in namespace %q", deleteConfigMap.Name, f.Namespace.Name) By(fmt.Sprintf("Updating configmap %v", updateConfigMap.Name)) updateConfigMap.ResourceVersion = "" // to force update delete(updateConfigMap.Data, "data-1") updateConfigMap.Data["data-3"] = "value-3" _, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(updateConfigMap) - Expect(err).NotTo(HaveOccurred(), "Failed to update configmap %q in namespace %q", updateConfigMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update configmap %q in namespace %q", updateConfigMap.Name, f.Namespace.Name) By(fmt.Sprintf("Creating configMap with name %s", createConfigMap.Name)) if createConfigMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(createConfigMap); err != nil { diff --git a/test/e2e/common/downwardapi_volume.go b/test/e2e/common/downwardapi_volume.go index f5d5f8d7a9f..fae20823558 100644 --- a/test/e2e/common/downwardapi_volume.go +++ b/test/e2e/common/downwardapi_volume.go @@ -163,7 +163,7 @@ var _ = Describe("[sig-storage] Downward API volume", func() { podClient.CreateSync(pod) pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to get pod %q", pod.Name) + framework.ExpectNoError(err, "Failed to get pod %q", pod.Name) Eventually(func() (string, error) { return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, containerName) diff --git a/test/e2e/common/expansion.go b/test/e2e/common/expansion.go index 1829468c515..1417da1001f 100644 --- a/test/e2e/common/expansion.go +++ b/test/e2e/common/expansion.go @@ -392,11 +392,11 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { By("waiting for pod running") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) - Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running") + framework.ExpectNoError(err, "while waiting for pod to be running") By("deleting the pod gracefully") err = framework.DeletePodWithWait(f, f.ClientSet, pod) - Expect(err).NotTo(HaveOccurred(), "failed to delete pod") + framework.ExpectNoError(err, "failed to delete pod") }) /* @@ -476,7 +476,7 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { By("waiting for pod running") err := framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) - Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running") + framework.ExpectNoError(err, "while waiting for pod to be running") By("creating a file in subpath") cmd := "touch /volume_mount/mypath/foo/test.log" @@ -499,11 +499,11 @@ var _ = framework.KubeDescribe("Variable Expansion", func() { By("waiting for annotated pod running") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) - Expect(err).NotTo(HaveOccurred(), "while waiting for annotated pod to be running") + framework.ExpectNoError(err, "while waiting for annotated pod to be running") By("deleting the pod gracefully") err = framework.DeletePodWithWait(f, f.ClientSet, pod) - Expect(err).NotTo(HaveOccurred(), "failed to delete pod") + framework.ExpectNoError(err, "failed to delete pod") }) /* diff --git a/test/e2e/common/init_container.go b/test/e2e/common/init_container.go index b84b8d58f7d..0b14698df26 100644 --- a/test/e2e/common/init_container.go +++ b/test/e2e/common/init_container.go @@ -91,7 +91,7 @@ var _ = framework.KubeDescribe("InitContainer [NodeConformance]", func() { e2elog.Logf("PodSpec: initContainers in spec.initContainers") startedPod := podClient.Create(pod) w, err := podClient.Watch(metav1.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") + framework.ExpectNoError(err, "error watching a pod") wr := watch.NewRecorder(w) ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), framework.PodStartTimeout) defer cancel() @@ -162,7 +162,7 @@ var _ = framework.KubeDescribe("InitContainer [NodeConformance]", func() { e2elog.Logf("PodSpec: initContainers in spec.initContainers") startedPod := podClient.Create(pod) w, err := podClient.Watch(metav1.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") + framework.ExpectNoError(err, "error watching a pod") wr := watch.NewRecorder(w) ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), framework.PodStartTimeout) defer cancel() @@ -234,7 +234,7 @@ var _ = framework.KubeDescribe("InitContainer [NodeConformance]", func() { e2elog.Logf("PodSpec: initContainers in spec.initContainers") startedPod := podClient.Create(pod) w, err := podClient.Watch(metav1.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") + framework.ExpectNoError(err, "error watching a pod") wr := watch.NewRecorder(w) ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), framework.PodStartTimeout) @@ -352,7 +352,7 @@ var _ = framework.KubeDescribe("InitContainer [NodeConformance]", func() { startedPod := podClient.Create(pod) w, err := podClient.Watch(metav1.SingleObject(startedPod.ObjectMeta)) - Expect(err).NotTo(HaveOccurred(), "error watching a pod") + framework.ExpectNoError(err, "error watching a pod") wr := watch.NewRecorder(w) ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), framework.PodStartTimeout) diff --git a/test/e2e/common/node_lease.go b/test/e2e/common/node_lease.go index e8ccc03a5e3..fb3d52137c6 100644 --- a/test/e2e/common/node_lease.go +++ b/test/e2e/common/node_lease.go @@ -144,7 +144,7 @@ var _ = framework.KubeDescribe("NodeLease", func() { }) // a timeout is acceptable, since it means we waited 5 minutes and didn't see any unwarranted node status updates if err != nil && err != wait.ErrWaitTimeout { - Expect(err).NotTo(HaveOccurred(), "error waiting for infrequent nodestatus update") + framework.ExpectNoError(err, "error waiting for infrequent nodestatus update") } By("verify node is still in ready status even though node status report is infrequent") diff --git a/test/e2e/common/privileged.go b/test/e2e/common/privileged.go index 5fe49f650b1..fd4e5ab4cdf 100644 --- a/test/e2e/common/privileged.go +++ b/test/e2e/common/privileged.go @@ -67,12 +67,12 @@ func (c *PrivilegedPodTestConfig) run(containerName string, expectSuccess bool) msg := fmt.Sprintf("cmd %v, stdout %q, stderr %q", cmd, stdout, stderr) if expectSuccess { - Expect(err).NotTo(HaveOccurred(), msg) + framework.ExpectNoError(err, msg) // We need to clean up the dummy link that was created, as it // leaks out into the node level -- yuck. _, _, err := c.f.ExecCommandInContainerWithFullOutput( c.privilegedPod, containerName, reverseCmd...) - Expect(err).NotTo(HaveOccurred(), + framework.ExpectNoError(err, fmt.Sprintf("could not remove dummy1 link: %v", err)) } else { Expect(err).To(HaveOccurred(), msg) diff --git a/test/e2e/common/projected_configmap.go b/test/e2e/common/projected_configmap.go index 02a0858fb83..df05c8eb217 100644 --- a/test/e2e/common/projected_configmap.go +++ b/test/e2e/common/projected_configmap.go @@ -188,7 +188,7 @@ var _ = Describe("[sig-storage] Projected configMap", func() { configMap.ResourceVersion = "" // to force update configMap.Data["data-1"] = "value-2" _, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(configMap) - Expect(err).NotTo(HaveOccurred(), "Failed to update configmap %q in namespace %q", configMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update configmap %q in namespace %q", configMap.Name, f.Namespace.Name) By("waiting to observe update in volume") Eventually(pollLogs, podLogTimeout, framework.Poll).Should(ContainSubstring("value-2")) @@ -374,14 +374,14 @@ var _ = Describe("[sig-storage] Projected configMap", func() { By(fmt.Sprintf("Deleting configmap %v", deleteConfigMap.Name)) err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(deleteConfigMap.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to delete configmap %q in namespace %q", deleteConfigMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to delete configmap %q in namespace %q", deleteConfigMap.Name, f.Namespace.Name) By(fmt.Sprintf("Updating configmap %v", updateConfigMap.Name)) updateConfigMap.ResourceVersion = "" // to force update delete(updateConfigMap.Data, "data-1") updateConfigMap.Data["data-3"] = "value-3" _, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(updateConfigMap) - Expect(err).NotTo(HaveOccurred(), "Failed to update configmap %q in namespace %q", updateConfigMap.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update configmap %q in namespace %q", updateConfigMap.Name, f.Namespace.Name) By(fmt.Sprintf("Creating configMap with name %s", createConfigMap.Name)) if createConfigMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(createConfigMap); err != nil { diff --git a/test/e2e/common/projected_downwardapi.go b/test/e2e/common/projected_downwardapi.go index b9c8c060e6f..d4153223c75 100644 --- a/test/e2e/common/projected_downwardapi.go +++ b/test/e2e/common/projected_downwardapi.go @@ -163,7 +163,7 @@ var _ = Describe("[sig-storage] Projected downwardAPI", func() { podClient.CreateSync(pod) pod, err := podClient.Get(pod.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to get pod %q", pod.Name) + framework.ExpectNoError(err, "Failed to get pod %q", pod.Name) Eventually(func() (string, error) { return framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, containerName) diff --git a/test/e2e/common/projected_secret.go b/test/e2e/common/projected_secret.go index f062cbf72ee..b861ef07a28 100644 --- a/test/e2e/common/projected_secret.go +++ b/test/e2e/common/projected_secret.go @@ -382,14 +382,14 @@ var _ = Describe("[sig-storage] Projected secret", func() { By(fmt.Sprintf("Deleting secret %v", deleteSecret.Name)) err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(deleteSecret.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to delete secret %q in namespace %q", deleteSecret.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to delete secret %q in namespace %q", deleteSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Updating secret %v", updateSecret.Name)) updateSecret.ResourceVersion = "" // to force update delete(updateSecret.Data, "data-1") updateSecret.Data["data-3"] = []byte("value-3") _, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Update(updateSecret) - Expect(err).NotTo(HaveOccurred(), "Failed to update secret %q in namespace %q", updateSecret.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update secret %q in namespace %q", updateSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Creating secret with name %s", createSecret.Name)) if createSecret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(createSecret); err != nil { diff --git a/test/e2e/common/runtimeclass.go b/test/e2e/common/runtimeclass.go index be38fc3ff27..10b054aa38d 100644 --- a/test/e2e/common/runtimeclass.go +++ b/test/e2e/common/runtimeclass.go @@ -38,6 +38,9 @@ const ( // PreconfiguredRuntimeHandler is the name of the runtime handler that is expected to be // preconfigured in the test environment. PreconfiguredRuntimeHandler = "test-handler" + // DockerRuntimeHandler is a hardcoded runtime handler that is accepted by dockershim, and + // treated equivalently to a nil runtime handler. + DockerRuntimeHandler = "docker" ) var _ = Describe("[sig-node] RuntimeClass", func() { @@ -59,9 +62,12 @@ var _ = Describe("[sig-node] RuntimeClass", func() { // This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes. It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() { // The built-in docker runtime does not support configuring runtime handlers. - framework.SkipIfContainerRuntimeIs("docker") + handler := PreconfiguredRuntimeHandler + if framework.TestContext.ContainerRuntime == "docker" { + handler = DockerRuntimeHandler + } - rcName := createRuntimeClass(f, "preconfigured-handler", PreconfiguredRuntimeHandler) + rcName := createRuntimeClass(f, "preconfigured-handler", handler) pod := createRuntimeClassPod(f, rcName) expectPodSuccess(f, pod) }) diff --git a/test/e2e/common/secrets_volume.go b/test/e2e/common/secrets_volume.go index d7193a2b350..05c2d11e1ee 100644 --- a/test/e2e/common/secrets_volume.go +++ b/test/e2e/common/secrets_volume.go @@ -347,14 +347,14 @@ var _ = Describe("[sig-storage] Secrets", func() { By(fmt.Sprintf("Deleting secret %v", deleteSecret.Name)) err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(deleteSecret.Name, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred(), "Failed to delete secret %q in namespace %q", deleteSecret.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to delete secret %q in namespace %q", deleteSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Updating secret %v", updateSecret.Name)) updateSecret.ResourceVersion = "" // to force update delete(updateSecret.Data, "data-1") updateSecret.Data["data-3"] = []byte("value-3") _, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Update(updateSecret) - Expect(err).NotTo(HaveOccurred(), "Failed to update secret %q in namespace %q", updateSecret.Name, f.Namespace.Name) + framework.ExpectNoError(err, "Failed to update secret %q in namespace %q", updateSecret.Name, f.Namespace.Name) By(fmt.Sprintf("Creating secret with name %s", createSecret.Name)) if createSecret, err = f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(createSecret); err != nil { diff --git a/test/e2e/common/volumes.go b/test/e2e/common/volumes.go index 8a76902976d..8981ba09313 100644 --- a/test/e2e/common/volumes.go +++ b/test/e2e/common/volumes.go @@ -49,7 +49,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework/volume" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) // These tests need privileged containers, which are disabled by default. Run @@ -130,7 +129,7 @@ var _ = Describe("[sig-storage] GCP Volumes", func() { defer func() { volume.TestCleanup(f, config) err := c.CoreV1().Endpoints(namespace.Name).Delete(name, nil) - Expect(err).NotTo(HaveOccurred(), "defer: Gluster delete endpoints failed") + framework.ExpectNoError(err, "defer: Gluster delete endpoints failed") }() tests := []volume.Test{ diff --git a/test/e2e/framework/auth/BUILD b/test/e2e/framework/auth/BUILD index f459c20f3eb..c1bbba2fa05 100644 --- a/test/e2e/framework/auth/BUILD +++ b/test/e2e/framework/auth/BUILD @@ -8,7 +8,6 @@ go_library( deps = [ "//staging/src/k8s.io/api/authorization/v1beta1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/test/e2e/framework/auth/helpers.go b/test/e2e/framework/auth/helpers.go index 5b3c9ac8d29..39161a73358 100644 --- a/test/e2e/framework/auth/helpers.go +++ b/test/e2e/framework/auth/helpers.go @@ -25,7 +25,6 @@ import ( "github.com/pkg/errors" authorizationv1beta1 "k8s.io/api/authorization/v1beta1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" @@ -68,15 +67,6 @@ func WaitForNamedAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviews err := wait.Poll(policyCachePollInterval, policyCachePollTimeout, func() (bool, error) { response, err := c.SubjectAccessReviews().Create(review) - // GKE doesn't enable the SAR endpoint. Without this endpoint, we cannot determine if the policy engine - // has adjusted as expected. In this case, simply wait one second and hope it's up to date - // TODO: Should have a check for the provider here but that introduces too tight of - // coupling with the `framework` package. See: https://github.com/kubernetes/kubernetes/issues/76726 - if apierrors.IsNotFound(err) { - logf("SubjectAccessReview endpoint is missing") - time.Sleep(1 * time.Second) - return true, nil - } if err != nil { return false, err } diff --git a/test/e2e/framework/endpoints/BUILD b/test/e2e/framework/endpoints/BUILD index 29eae19fe48..2310197e48c 100644 --- a/test/e2e/framework/endpoints/BUILD +++ b/test/e2e/framework/endpoints/BUILD @@ -2,14 +2,21 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["wait.go"], + srcs = [ + "ports.go", + "wait.go", + ], importpath = "k8s.io/kubernetes/test/e2e/framework/endpoints", visibility = ["//visibility:public"], deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/log:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", ], ) diff --git a/test/e2e/framework/endpoints/ports.go b/test/e2e/framework/endpoints/ports.go new file mode 100644 index 00000000000..c358e507fff --- /dev/null +++ b/test/e2e/framework/endpoints/ports.go @@ -0,0 +1,136 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This soak tests places a specified number of pods on each node and then +repeatedly sends queries to a service running on these pods via +a serivce +*/ + +package endpoints + +import ( + "fmt" + "sort" + "time" + + "github.com/onsi/ginkgo" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" +) + +// ServiceStartTimeout is how long to wait for a service endpoint to be resolvable. +const ServiceStartTimeout = 3 * time.Minute + +// PortsByPodName is a map that maps pod name to container ports. +type PortsByPodName map[string][]int + +// PortsByPodUID is a map that maps pod UID to container ports. +type PortsByPodUID map[types.UID][]int + +// GetContainerPortsByPodUID returns a PortsByPodUID map on the given endpoints. +func GetContainerPortsByPodUID(ep *v1.Endpoints) PortsByPodUID { + m := PortsByPodUID{} + for _, ss := range ep.Subsets { + for _, port := range ss.Ports { + for _, addr := range ss.Addresses { + containerPort := port.Port + if _, ok := m[addr.TargetRef.UID]; !ok { + m[addr.TargetRef.UID] = make([]int, 0) + } + m[addr.TargetRef.UID] = append(m[addr.TargetRef.UID], int(containerPort)) + } + } + } + return m +} + +func translatePodNameToUID(c clientset.Interface, ns string, expectedEndpoints PortsByPodName) (PortsByPodUID, error) { + portsByUID := make(PortsByPodUID) + for name, portList := range expectedEndpoints { + pod, err := c.CoreV1().Pods(ns).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err) + } + portsByUID[pod.ObjectMeta.UID] = portList + } + return portsByUID, nil +} + +func validatePorts(ep PortsByPodUID, expectedEndpoints PortsByPodUID) error { + if len(ep) != len(expectedEndpoints) { + // should not happen because we check this condition before + return fmt.Errorf("invalid number of endpoints got %v, expected %v", ep, expectedEndpoints) + } + for podUID := range expectedEndpoints { + if _, ok := ep[podUID]; !ok { + return fmt.Errorf("endpoint %v not found", podUID) + } + if len(ep[podUID]) != len(expectedEndpoints[podUID]) { + return fmt.Errorf("invalid list of ports for uid %v. Got %v, expected %v", podUID, ep[podUID], expectedEndpoints[podUID]) + } + sort.Ints(ep[podUID]) + sort.Ints(expectedEndpoints[podUID]) + for index := range ep[podUID] { + if ep[podUID][index] != expectedEndpoints[podUID][index] { + return fmt.Errorf("invalid list of ports for uid %v. Got %v, expected %v", podUID, ep[podUID], expectedEndpoints[podUID]) + } + } + } + return nil +} + +// ValidateEndpointsPorts validates that the given service exists and is served by the given expectedEndpoints. +func ValidateEndpointsPorts(c clientset.Interface, namespace, serviceName string, expectedEndpoints PortsByPodName) error { + ginkgo.By(fmt.Sprintf("waiting up to %v for service %s in namespace %s to expose endpoints %v", ServiceStartTimeout, serviceName, namespace, expectedEndpoints)) + i := 1 + for start := time.Now(); time.Since(start) < ServiceStartTimeout; time.Sleep(1 * time.Second) { + ep, err := c.CoreV1().Endpoints(namespace).Get(serviceName, metav1.GetOptions{}) + if err != nil { + e2elog.Logf("Get endpoints failed (%v elapsed, ignoring for 5s): %v", time.Since(start), err) + continue + } + portsByPodUID := GetContainerPortsByPodUID(ep) + expectedPortsByPodUID, err := translatePodNameToUID(c, namespace, expectedEndpoints) + if err != nil { + return err + } + if len(portsByPodUID) == len(expectedEndpoints) { + err := validatePorts(portsByPodUID, expectedPortsByPodUID) + if err != nil { + return err + } + e2elog.Logf("successfully validated that service %s in namespace %s exposes endpoints %v (%v elapsed)", + serviceName, namespace, expectedEndpoints, time.Since(start)) + return nil + } + if i%5 == 0 { + e2elog.Logf("Unexpected endpoints: found %v, expected %v (%v elapsed, will retry)", portsByPodUID, expectedEndpoints, time.Since(start)) + } + i++ + } + if pods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil { + for _, pod := range pods.Items { + e2elog.Logf("Pod %s\t%s\t%s\t%s", pod.Namespace, pod.Name, pod.Spec.NodeName, pod.DeletionTimestamp) + } + } else { + e2elog.Logf("Can't list pod debug info: %v", err) + } + return fmt.Errorf("Timed out waiting for service %s in namespace %s to expose endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, ServiceStartTimeout) +} diff --git a/test/e2e/framework/metrics/cluster_autoscaler_metrics.go b/test/e2e/framework/metrics/cluster_autoscaler_metrics.go index 0863890fbdb..8547e147479 100644 --- a/test/e2e/framework/metrics/cluster_autoscaler_metrics.go +++ b/test/e2e/framework/metrics/cluster_autoscaler_metrics.go @@ -16,7 +16,7 @@ limitations under the License. package metrics -// ClusterAutoscalerMetrics is metrics for cluster autoscaller +// ClusterAutoscalerMetrics is metrics for cluster autoscaler type ClusterAutoscalerMetrics Metrics // Equal returns true if all metrics are the same as the arguments. diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index f2d99b54094..434f199dab9 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" @@ -1238,104 +1237,6 @@ func UpdateService(c clientset.Interface, namespace, serviceName string, update return service, err } -// GetContainerPortsByPodUID returns a PortsByPodUID map on the given endpoints. -func GetContainerPortsByPodUID(endpoints *v1.Endpoints) PortsByPodUID { - m := PortsByPodUID{} - for _, ss := range endpoints.Subsets { - for _, port := range ss.Ports { - for _, addr := range ss.Addresses { - containerPort := port.Port - if _, ok := m[addr.TargetRef.UID]; !ok { - m[addr.TargetRef.UID] = make([]int, 0) - } - m[addr.TargetRef.UID] = append(m[addr.TargetRef.UID], int(containerPort)) - } - } - } - return m -} - -// PortsByPodName maps pod name to ports. -type PortsByPodName map[string][]int - -// PortsByPodUID maps UID to ports. -type PortsByPodUID map[types.UID][]int - -func translatePodNameToUIDOrFail(c clientset.Interface, ns string, expectedEndpoints PortsByPodName) PortsByPodUID { - portsByUID := make(PortsByPodUID) - - for name, portList := range expectedEndpoints { - pod, err := c.CoreV1().Pods(ns).Get(name, metav1.GetOptions{}) - if err != nil { - Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err) - } - portsByUID[pod.ObjectMeta.UID] = portList - } - // Logf("successfully translated pod names to UIDs: %v -> %v on namespace %s", expectedEndpoints, portsByUID, ns) - return portsByUID -} - -func validatePortsOrFail(endpoints PortsByPodUID, expectedEndpoints PortsByPodUID) { - if len(endpoints) != len(expectedEndpoints) { - // should not happen because we check this condition before - Failf("invalid number of endpoints got %v, expected %v", endpoints, expectedEndpoints) - } - for podUID := range expectedEndpoints { - if _, ok := endpoints[podUID]; !ok { - Failf("endpoint %v not found", podUID) - } - if len(endpoints[podUID]) != len(expectedEndpoints[podUID]) { - Failf("invalid list of ports for uid %v. Got %v, expected %v", podUID, endpoints[podUID], expectedEndpoints[podUID]) - } - sort.Ints(endpoints[podUID]) - sort.Ints(expectedEndpoints[podUID]) - for index := range endpoints[podUID] { - if endpoints[podUID][index] != expectedEndpoints[podUID][index] { - Failf("invalid list of ports for uid %v. Got %v, expected %v", podUID, endpoints[podUID], expectedEndpoints[podUID]) - } - } - } -} - -// ValidateEndpointsOrFail validates that the given service exists and is served by the given expectedEndpoints. -func ValidateEndpointsOrFail(c clientset.Interface, namespace, serviceName string, expectedEndpoints PortsByPodName) { - ginkgo.By(fmt.Sprintf("waiting up to %v for service %s in namespace %s to expose endpoints %v", ServiceStartTimeout, serviceName, namespace, expectedEndpoints)) - i := 1 - for start := time.Now(); time.Since(start) < ServiceStartTimeout; time.Sleep(1 * time.Second) { - endpoints, err := c.CoreV1().Endpoints(namespace).Get(serviceName, metav1.GetOptions{}) - if err != nil { - Logf("Get endpoints failed (%v elapsed, ignoring for 5s): %v", time.Since(start), err) - continue - } - // Logf("Found endpoints %v", endpoints) - - portsByPodUID := GetContainerPortsByPodUID(endpoints) - // Logf("Found port by pod UID %v", portsByPodUID) - - expectedPortsByPodUID := translatePodNameToUIDOrFail(c, namespace, expectedEndpoints) - if len(portsByPodUID) == len(expectedEndpoints) { - validatePortsOrFail(portsByPodUID, expectedPortsByPodUID) - Logf("successfully validated that service %s in namespace %s exposes endpoints %v (%v elapsed)", - serviceName, namespace, expectedEndpoints, time.Since(start)) - return - } - - if i%5 == 0 { - Logf("Unexpected endpoints: found %v, expected %v (%v elapsed, will retry)", portsByPodUID, expectedEndpoints, time.Since(start)) - } - i++ - } - - if pods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil { - for _, pod := range pods.Items { - Logf("Pod %s\t%s\t%s\t%s", pod.Namespace, pod.Name, pod.Spec.NodeName, pod.DeletionTimestamp) - } - } else { - Logf("Can't list pod debug info: %v", err) - } - Failf("Timed out waiting for service %s in namespace %s to expose endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, ServiceStartTimeout) -} - // StartServeHostnameService creates a replication controller that serves its // hostname and a service on top of it. func StartServeHostnameService(c clientset.Interface, svc *v1.Service, ns string, replicas int) ([]string, string, error) { diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index f740c529be0..2880d843f03 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -33,6 +33,7 @@ go_library( "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/auth:go_default_library", + "//test/e2e/framework/endpoints:go_default_library", "//test/e2e/framework/job:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index 6249da57d74..157f6d922ec 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -40,8 +40,6 @@ import ( "time" "github.com/elazarl/goproxy" - "sigs.k8s.io/yaml" - v1 "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -60,6 +58,7 @@ import ( commonutils "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/auth" + e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints" jobutil "k8s.io/kubernetes/test/e2e/framework/job" e2elog "k8s.io/kubernetes/test/e2e/framework/log" "k8s.io/kubernetes/test/e2e/framework/testfiles" @@ -67,6 +66,7 @@ import ( testutils "k8s.io/kubernetes/test/utils" "k8s.io/kubernetes/test/utils/crd" uexec "k8s.io/utils/exec" + "sigs.k8s.io/yaml" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -1075,7 +1075,7 @@ metadata: }) validateService := func(name string, servicePort int, timeout time.Duration) { err := wait.Poll(framework.Poll, timeout, func() (bool, error) { - endpoints, err := c.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{}) + ep, err := c.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{}) if err != nil { // log the real error e2elog.Logf("Get endpoints failed (interval %v): %v", framework.Poll, err) @@ -1089,7 +1089,7 @@ metadata: return false, err } - uidToPort := framework.GetContainerPortsByPodUID(endpoints) + uidToPort := e2eendpoints.GetContainerPortsByPodUID(ep) if len(uidToPort) == 0 { e2elog.Logf("No endpoint found, retrying") return false, nil diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 187b07122a8..97ca2675c41 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -37,6 +37,7 @@ import ( cloudprovider "k8s.io/cloud-provider" "k8s.io/kubernetes/pkg/controller/endpoint" "k8s.io/kubernetes/test/e2e/framework" + e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints" e2elog "k8s.io/kubernetes/test/e2e/framework/log" "k8s.io/kubernetes/test/e2e/framework/providers/gce" e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" @@ -138,7 +139,8 @@ var _ = SIGDescribe("Services", func() { framework.ExpectNoError(err, "failed to create service with ServicePorts in namespace: %s", ns) - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) names := map[string]bool{} defer func() { @@ -153,19 +155,23 @@ var _ = SIGDescribe("Services", func() { framework.CreatePodOrFail(cs, ns, name1, labels, []v1.ContainerPort{{ContainerPort: 80}}) names[name1] = true - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{name1: {80}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{name1: {80}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.CreatePodOrFail(cs, ns, name2, labels, []v1.ContainerPort{{ContainerPort: 80}}) names[name2] = true - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{name1: {80}, name2: {80}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{name1: {80}, name2: {80}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.DeletePodOrFail(cs, ns, name1) delete(names, name1) - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{name2: {80}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{name2: {80}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.DeletePodOrFail(cs, ns, name2) delete(names, name2) - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) }) /* @@ -206,7 +212,8 @@ var _ = SIGDescribe("Services", func() { framework.ExpectNoError(err, "failed to create service with ServicePorts in namespace: %s", ns) port1 := 100 port2 := 101 - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) names := map[string]bool{} defer func() { @@ -234,19 +241,23 @@ var _ = SIGDescribe("Services", func() { framework.CreatePodOrFail(cs, ns, podname1, labels, containerPorts1) names[podname1] = true - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{podname1: {port1}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{podname1: {port1}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.CreatePodOrFail(cs, ns, podname2, labels, containerPorts2) names[podname2] = true - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{podname1: {port1}, podname2: {port2}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{podname1: {port1}, podname2: {port2}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.DeletePodOrFail(cs, ns, podname1) delete(names, podname1) - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{podname2: {port2}}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{podname2: {port2}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) framework.DeletePodOrFail(cs, ns, podname2) delete(names, podname2) - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{}) + err = e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) }) ginkgo.It("should preserve source pod IP for traffic thru service cluster IP", func() { @@ -297,7 +308,8 @@ var _ = SIGDescribe("Services", func() { }() // Waiting for service to expose endpoint. - framework.ValidateEndpointsOrFail(cs, ns, serviceName, framework.PortsByPodName{serverPodName: {servicePort}}) + err := e2eendpoints.ValidateEndpointsPorts(cs, ns, serviceName, e2eendpoints.PortsByPodName{serverPodName: {servicePort}}) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) ginkgo.By("Retrieve sourceip from a pod on the same node") sourceIP1, execPodIP1 := execSourceipTest(f, cs, ns, node1.Name, serviceIP, servicePort) diff --git a/test/e2e/scalability/load.go b/test/e2e/scalability/load.go index 69d7e87ebc0..512927abc37 100644 --- a/test/e2e/scalability/load.go +++ b/test/e2e/scalability/load.go @@ -347,7 +347,7 @@ func createClients(numberOfClients int) ([]clientset.Interface, []scaleclient.Sc for i := 0; i < numberOfClients; i++ { config, err := framework.LoadConfig() - Expect(err).NotTo(HaveOccurred()) + framework.ExpectNoError(err) config.QPS = 100 config.Burst = 200 if framework.TestContext.KubeAPIContentType != "" { diff --git a/test/e2e/scheduling/equivalence_cache_predicates.go b/test/e2e/scheduling/equivalence_cache_predicates.go index d7f6b0cf557..b5d5642deff 100644 --- a/test/e2e/scheduling/equivalence_cache_predicates.go +++ b/test/e2e/scheduling/equivalence_cache_predicates.go @@ -62,7 +62,7 @@ var _ = framework.KubeDescribe("EquivalenceCache [Serial]", func() { // cannot be run in parallel with any other test that touches Nodes or Pods. // It is so because we need to have precise control on what's running in the cluster. systemPods, err := framework.GetPodsInNamespace(cs, ns, map[string]string{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) systemPodsNo = 0 for _, pod := range systemPods { if !masterNodes.Has(pod.Spec.NodeName) && pod.DeletionTimestamp == nil { @@ -71,7 +71,7 @@ var _ = framework.KubeDescribe("EquivalenceCache [Serial]", func() { } err = framework.WaitForPodsRunningReady(cs, api.NamespaceSystem, int32(systemPodsNo), int32(systemPodsNo), framework.PodReadyBeforeTimeout, map[string]string{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) for _, node := range nodeList.Items { e2elog.Logf("\nLogging pods the kubelet thinks is on node %v before test", node.Name) diff --git a/test/e2e/scheduling/limit_range.go b/test/e2e/scheduling/limit_range.go index 37b38d9ebff..528229c2ad0 100644 --- a/test/e2e/scheduling/limit_range.go +++ b/test/e2e/scheduling/limit_range.go @@ -58,18 +58,18 @@ var _ = SIGDescribe("LimitRange", func() { selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": limitRange.Name})) options := metav1.ListOptions{LabelSelector: selector.String()} limitRanges, err := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).List(options) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "failed to query for limitRanges") + framework.ExpectNoError(err, "failed to query for limitRanges") gomega.Expect(len(limitRanges.Items)).To(gomega.Equal(0)) options = metav1.ListOptions{ LabelSelector: selector.String(), ResourceVersion: limitRanges.ListMeta.ResourceVersion, } w, err := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Watch(metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "failed to set up watch") + framework.ExpectNoError(err, "failed to set up watch") ginkgo.By("Submitting a LimitRange") limitRange, err = f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Create(limitRange) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Verifying LimitRange creation was observed") select { @@ -83,37 +83,37 @@ var _ = SIGDescribe("LimitRange", func() { ginkgo.By("Fetching the LimitRange to ensure it has proper values") limitRange, err = f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Get(limitRange.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) expected := v1.ResourceRequirements{Requests: defaultRequest, Limits: defaultLimit} actual := v1.ResourceRequirements{Requests: limitRange.Spec.Limits[0].DefaultRequest, Limits: limitRange.Spec.Limits[0].Default} err = equalResourceRequirement(expected, actual) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Creating a Pod with no resource requirements") pod := f.NewTestPod("pod-no-resources", v1.ResourceList{}, v1.ResourceList{}) pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Ensuring Pod has resource requirements applied from LimitRange") pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) for i := range pod.Spec.Containers { err = equalResourceRequirement(expected, pod.Spec.Containers[i].Resources) if err != nil { // Print the pod to help in debugging. e2elog.Logf("Pod %+v does not have the expected requirements", pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } } ginkgo.By("Creating a Pod with partial resource requirements") pod = f.NewTestPod("pod-partial-resources", getResourceList("", "150Mi", "150Gi"), getResourceList("300m", "", "")) pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Ensuring Pod has merged resource requirements applied from LimitRange") pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // This is an interesting case, so it's worth a comment // If you specify a Limit, and no Request, the Limit will default to the Request // This means that the LimitRange.DefaultRequest will ONLY take affect if a container.resources.limit is not supplied @@ -123,7 +123,7 @@ var _ = SIGDescribe("LimitRange", func() { if err != nil { // Print the pod to help in debugging. e2elog.Logf("Pod %+v does not have the expected requirements", pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } } @@ -141,19 +141,20 @@ var _ = SIGDescribe("LimitRange", func() { newMin := getResourceList("9m", "49Mi", "49Gi") limitRange.Spec.Limits[0].Min = newMin limitRange, err = f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Update(limitRange) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Verifying LimitRange updating is effective") - gomega.Expect(wait.Poll(time.Second*2, time.Second*20, func() (bool, error) { + err = wait.Poll(time.Second*2, time.Second*20, func() (bool, error) { limitRange, err = f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Get(limitRange.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) return reflect.DeepEqual(limitRange.Spec.Limits[0].Min, newMin), nil - })).NotTo(gomega.HaveOccurred()) + }) + framework.ExpectNoError(err) ginkgo.By("Creating a Pod with less than former min resources") pod = f.NewTestPod(podName, getResourceList("10m", "50Mi", "50Gi"), v1.ResourceList{}) pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Failing to create a Pod with more than max resources") pod = f.NewTestPod(podName, getResourceList("600m", "600Mi", "600Gi"), v1.ResourceList{}) @@ -162,10 +163,10 @@ var _ = SIGDescribe("LimitRange", func() { ginkgo.By("Deleting a LimitRange") err = f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).Delete(limitRange.Name, metav1.NewDeleteOptions(30)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By("Verifying the LimitRange was deleted") - gomega.Expect(wait.Poll(time.Second*5, time.Second*30, func() (bool, error) { + err = wait.Poll(time.Second*5, time.Second*30, func() (bool, error) { selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": limitRange.Name})) options := metav1.ListOptions{LabelSelector: selector.String()} limitRanges, err := f.ClientSet.CoreV1().LimitRanges(f.Namespace.Name).List(options) @@ -190,12 +191,13 @@ var _ = SIGDescribe("LimitRange", func() { return false, nil - })).NotTo(gomega.HaveOccurred(), "kubelet never observed the termination notice") + }) + framework.ExpectNoError(err, "kubelet never observed the termination notice") ginkgo.By("Creating a Pod with more than former max resources") pod = f.NewTestPod(podName+"2", getResourceList("600m", "600Mi", "600Gi"), v1.ResourceList{}) pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) }) }) diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 24420b85468..b7b5a5a7883 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -713,7 +713,7 @@ func WaitForSchedulerAfterAction(f *framework.Framework, action common.Action, n predicate = scheduleSuccessEvent(ns, podName, "" /* any node */) } success, err := common.ObserveEventAfterAction(f, predicate, action) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(success).To(gomega.Equal(true)) } diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go index 838caa754f5..18d37cd08ca 100644 --- a/test/e2e/scheduling/priorities.go +++ b/test/e2e/scheduling/priorities.go @@ -82,7 +82,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { err := framework.CheckTestingNSDeletedExcept(cs, ns) framework.ExpectNoError(err) err = framework.WaitForPodsRunningReady(cs, metav1.NamespaceSystem, int32(systemPodsNo), 0, framework.PodReadyBeforeTimeout, map[string]string{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) }) ginkgo.It("Pod should be scheduled to node that don't match the PodAntiAffinity terms", func() { @@ -191,7 +191,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { return node.Annotations[v1.PreferAvoidPodsAnnotationKey] == string(val) } success, err := common.ObserveNodeUpdateAfterAction(f, nodeName, predicate, action) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(success).To(gomega.Equal(true)) defer framework.RemoveAvoidPodsOffNode(cs, nodeName) @@ -202,7 +202,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { testPods, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{ LabelSelector: "name=scheduler-priority-avoid-pod", }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("Verify the pods should not scheduled to the node: %s", nodeName)) for _, pod := range testPods.Items { gomega.Expect(pod.Spec.NodeName).NotTo(gomega.Equal(nodeName)) @@ -235,7 +235,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { ginkgo.By("Pod should prefer scheduled to the node don't have the taint.") tolePod, err := cs.CoreV1().Pods(ns).Get(tolerationPodName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(tolePod.Spec.NodeName).To(gomega.Equal(nodeName)) ginkgo.By("Trying to apply 10 taint on the first node.") @@ -255,7 +255,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { ginkgo.By("Pod should prefer scheduled to the node that pod can tolerate.") tolePod, err = cs.CoreV1().Pods(ns).Get(tolerationPodName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(tolePod.Spec.NodeName).To(gomega.Equal(nodeName)) }) }) @@ -400,7 +400,7 @@ func createRC(ns, rsName string, replicas int32, rcPodLabels map[string]string, }, } rc, err := f.ClientSet.CoreV1().ReplicationControllers(ns).Create(rc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) return rc } diff --git a/test/e2e/scheduling/ubernetes_lite.go b/test/e2e/scheduling/ubernetes_lite.go index c907155c94d..7cf64901cad 100644 --- a/test/e2e/scheduling/ubernetes_lite.go +++ b/test/e2e/scheduling/ubernetes_lite.go @@ -43,7 +43,7 @@ var _ = SIGDescribe("Multi-AZ Clusters", func() { framework.SkipUnlessProviderIs("gce", "gke", "aws") if zoneCount <= 0 { zoneCount, err = getZoneCount(f.ClientSet) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } ginkgo.By(fmt.Sprintf("Checking for multi-zone cluster. Zone count = %d", zoneCount)) msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount) @@ -80,7 +80,7 @@ func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) }, } _, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(serviceSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Now create some pods behind the service podSpec := &v1.Pod{ @@ -107,11 +107,11 @@ func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) // Wait for all of them to be scheduled selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName})) pods, err := framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, selector) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Now make sure they're spread across zones zoneNames, err := framework.GetClusterZones(f.ClientSet) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(checkZoneSpreading(f.ClientSet, pods, zoneNames.List())).To(gomega.Equal(true)) } @@ -139,7 +139,7 @@ func getZoneCount(c clientset.Interface) (int, error) { func getZoneNameForPod(c clientset.Interface, pod v1.Pod) (string, error) { ginkgo.By(fmt.Sprintf("Getting zone name for pod %s, on node %s", pod.Name, pod.Spec.NodeName)) node, err := c.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) return getZoneNameForNode(*node) } @@ -155,7 +155,7 @@ func checkZoneSpreading(c clientset.Interface, pods *v1.PodList, zoneNames []str continue } zoneName, err := getZoneNameForPod(c, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) podsPerZone[zoneName] = podsPerZone[zoneName] + 1 } minPodsPerZone := math.MaxInt32 @@ -205,7 +205,7 @@ func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string) { }, }, }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Cleanup the replication controller when we are done. defer func() { // Resize the replication controller to zero to get rid of pods. @@ -216,15 +216,15 @@ func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string) { // List the pods, making sure we observe all the replicas. selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": name})) pods, err := framework.PodsCreated(f.ClientSet, f.Namespace.Name, name, replicaCount) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Wait for all of them to be scheduled ginkgo.By(fmt.Sprintf("Waiting for %d replicas of %s to be scheduled. Selector: %v", replicaCount, name, selector)) pods, err = framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, selector) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Now make sure they're spread across zones zoneNames, err := framework.GetClusterZones(f.ClientSet) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) gomega.Expect(checkZoneSpreading(f.ClientSet, pods, zoneNames.List())).To(gomega.Equal(true)) } diff --git a/test/e2e/scheduling/ubernetes_lite_volumes.go b/test/e2e/scheduling/ubernetes_lite_volumes.go index 38ef8d9b3f2..efabfff5807 100644 --- a/test/e2e/scheduling/ubernetes_lite_volumes.go +++ b/test/e2e/scheduling/ubernetes_lite_volumes.go @@ -42,7 +42,7 @@ var _ = SIGDescribe("Multi-AZ Cluster Volumes [sig-storage]", func() { framework.SkipUnlessProviderIs("gce", "gke") if zoneCount <= 0 { zoneCount, err = getZoneCount(f.ClientSet) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } ginkgo.By(fmt.Sprintf("Checking for multi-zone cluster. Zone count = %d", zoneCount)) msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount) @@ -61,17 +61,17 @@ var _ = SIGDescribe("Multi-AZ Cluster Volumes [sig-storage]", func() { // OnlyAllowNodeZones tests that GetAllCurrentZones returns only zones with Nodes func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) { gceCloud, err := gce.GetGCECloud() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Get all the zones that the nodes are in expectedZones, err := gceCloud.GetAllZonesFromCloudProvider() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) e2elog.Logf("Expected zones: %v", expectedZones) // Get all the zones in this current region region := gceCloud.Region() allZonesInRegion, err := gceCloud.ListZonesInRegion(region) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) var extraZone string for _, zone := range allZonesInRegion { @@ -117,13 +117,13 @@ func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) { } err = gceCloud.InsertInstance(project, zone, rb) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) defer func() { // Teardown of the compute instance e2elog.Logf("Deleting compute resource: %v", name) err := gceCloud.DeleteInstance(project, zone, name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) }() ginkgo.By("Creating zoneCount+1 PVCs and making sure PDs are only provisioned in zones with nodes") @@ -136,7 +136,7 @@ func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) { for index := 1; index <= zoneCount+1; index++ { pvc := newNamedDefaultClaim(ns, index) pvc, err = framework.CreatePVC(c, ns, pvc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) pvcList = append(pvcList, pvc) // Defer the cleanup @@ -152,7 +152,7 @@ func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) { // Wait for all claims bound for _, claim := range pvcList { err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } pvZones := sets.NewString() @@ -160,11 +160,12 @@ func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) { for _, claim := range pvcList { // Get a new copy of the claim to have all fields populated claim, err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) // Get the related PV pv, err := c.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) pvZone, ok := pv.ObjectMeta.Labels[v1.LabelZoneFailureDomain] gomega.Expect(ok).To(gomega.BeTrue(), "PV has no LabelZone to be found") @@ -188,7 +189,7 @@ func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) ns := f.Namespace.Name zones, err := framework.GetClusterZones(c) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) zonelist := zones.List() ginkgo.By("Creating static PVs across zones") configs := make([]*staticPVTestConfig, podCount) @@ -205,14 +206,14 @@ func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) framework.WaitForPodNoLongerRunningInNamespace(c, config.pod.Name, ns) framework.PVPVCCleanup(c, ns, config.pv, config.pvc) err = framework.DeletePVSource(config.pvSource) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } }() for i, config := range configs { zone := zonelist[i%len(zones)] config.pvSource, err = framework.CreatePVSource(zone) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) pvConfig := framework.PersistentVolumeConfig{ NamePrefix: "multizone-pv", @@ -223,7 +224,7 @@ func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) pvcConfig := framework.PersistentVolumeClaimConfig{StorageClassName: &className} config.pv, config.pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } ginkgo.By("Waiting for all PVCs to be bound") @@ -235,13 +236,13 @@ func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) for _, config := range configs { podConfig := framework.MakePod(ns, nil, []*v1.PersistentVolumeClaim{config.pvc}, false, "") config.pod, err = c.CoreV1().Pods(ns).Create(podConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } ginkgo.By("Waiting for all pods to be running") for _, config := range configs { err = framework.WaitForPodRunningInNamespace(c, config.pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.ExpectNoError(err) } } diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index b4e0c88a574..12537856b2d 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -51,7 +51,9 @@ import ( ) const ( + // GCEPDCSIProvisionerName is the name of GCE Persistent Disk CSI provisioner GCEPDCSIProvisionerName = "pd.csi.storage.gke.io" + // GCEPDCSIZoneTopologyKey is the key of GCE Persistent Disk CSI zone topology GCEPDCSIZoneTopologyKey = "topology.gke.io/zone" ) diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 5bcd3c3426a..e2329110fbb 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -1207,7 +1207,7 @@ func (g *gcePdDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTest return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix) } -func (h *gcePdDriver) GetClaimSize() string { +func (g *gcePdDriver) GetClaimSize() string { return "5Gi" } @@ -1652,6 +1652,7 @@ var _ testsuites.TestDriver = &localDriver{} var _ testsuites.PreprovisionedVolumeTestDriver = &localDriver{} var _ testsuites.PreprovisionedPVTestDriver = &localDriver{} +// InitLocalDriverWithVolumeType initializes the local driver based on the volume type. func InitLocalDriverWithVolumeType(volumeType utils.LocalVolumeType) func() testsuites.TestDriver { maxFileSize := defaultLocalVolumeMaxFileSize if maxFileSizeByVolType, ok := localVolumeMaxFileSizes[volumeType]; ok { diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index f708794356f..d16a19222e0 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -512,11 +512,12 @@ func getVolumeOpCounts(c clientset.Interface, pluginName string) opCounts { metricsGrabber, err := metrics.NewMetricsGrabber(c, nil, true, false, true, false, false) if err != nil { - framework.Failf("Error creating metrics grabber : %v", err) + framework.ExpectNoError(err, "Error creating metrics grabber: %v", err) } if !metricsGrabber.HasRegisteredMaster() { - framework.Skipf("Environment does not support getting controller-manager metrics - skipping") + e2elog.Logf("Warning: Environment does not support getting controller-manager metrics") + return opCounts{} } controllerMetrics, err := metricsGrabber.GrabFromControllerManager() diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index 69290a485da..99179d35e9d 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -118,12 +118,11 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T // Now do the more expensive test initialization. l.config, l.testCleanup = driver.PrepareTest(f) + l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName) l.resource = createGenericVolumeTestResource(driver, l.config, pattern) if l.resource.volSource == nil { framework.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name) } - - l.intreeOps, l.migratedOps = getMigrationVolumeOpCounts(f.ClientSet, dInfo.InTreePluginName) } cleanup := func() { diff --git a/test/e2e/windows/gmsa.go b/test/e2e/windows/gmsa.go index 687a600f257..21aef8bd5e4 100644 --- a/test/e2e/windows/gmsa.go +++ b/test/e2e/windows/gmsa.go @@ -46,24 +46,31 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func( container2Name := "container2" container2Domain := "contoso.org" - containers := make([]corev1.Container, 2) - for i, name := range []string{container1Name, container2Name} { - containers[i] = corev1.Container{ - Name: name, - Image: imageutils.GetPauseImageName(), - } - } - pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, - Annotations: map[string]string{ - "pod.alpha.windows.kubernetes.io/gmsa-credential-spec": generateDummyCredSpecs(podDomain), - container2Name + ".container.alpha.windows.kubernetes.io/gmsa-credential-spec": generateDummyCredSpecs(container2Domain), - }, }, Spec: corev1.PodSpec{ - Containers: containers, + Containers: []corev1.Container{ + { + Name: container1Name, + Image: imageutils.GetPauseImageName(), + }, + { + Name: container2Name, + Image: imageutils.GetPauseImageName(), + SecurityContext: &corev1.SecurityContext{ + WindowsOptions: &corev1.WindowsSecurityContextOptions{ + GMSACredentialSpec: generateDummyCredSpecs(container2Domain), + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + WindowsOptions: &corev1.WindowsSecurityContextOptions{ + GMSACredentialSpec: generateDummyCredSpecs(podDomain), + }, + }, }, } @@ -108,10 +115,10 @@ var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA [Slow]", func( }) }) -func generateDummyCredSpecs(domain string) string { +func generateDummyCredSpecs(domain string) *string { shortName := strings.ToUpper(strings.Split(domain, ".")[0]) - return fmt.Sprintf(`{ + credSpecs := fmt.Sprintf(`{ "ActiveDirectoryConfig":{ "GroupManagedServiceAccounts":[ { @@ -136,4 +143,6 @@ func generateDummyCredSpecs(domain string) string { "Sid":"S-1-5-21-2126729477-2524175714-3194792973" } }`, shortName, domain, domain, domain, shortName) + + return &credSpecs } diff --git a/test/e2e_node/conformance/run_test.sh b/test/e2e_node/conformance/run_test.sh index a83581a7cdb..cd7e2684fcc 100755 --- a/test/e2e_node/conformance/run_test.sh +++ b/test/e2e_node/conformance/run_test.sh @@ -176,7 +176,6 @@ if [ ! -z $pid ]; then fi volume_stats_agg_period=10s -allow_privileged=true serialize_image_pulls=false config_dir=`mktemp -d` file_check_frequency=10s @@ -184,7 +183,6 @@ pod_cidr=10.100.0.0/24 log_level=4 start_kubelet --kubeconfig ${KUBELET_KUBECONFIG} \ --volume-stats-agg-period $volume_stats_agg_period \ - --allow-privileged=$allow_privileged \ --serialize-image-pulls=$serialize_image_pulls \ --pod-manifest-path $config_dir \ --file-check-frequency $file_check_frequency \ diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 407a31eaec1..a44f1fc1475 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -21,6 +21,7 @@ limitations under the License. package main import ( + "context" "flag" "fmt" "io/ioutil" @@ -152,6 +153,8 @@ type GCEImage struct { // Defaults to using only the latest image. Acceptable values are [0, # of images that match the regex). // If the number of existing previous images is lesser than what is desired, the test will use that is available. PreviousImages int `json:"previous_images,omitempty"` + // ImageFamily is the image family to use. The latest image from the image family will be used. + ImageFamily string `json:"image_family,omitempty"` Machine string `json:"machine,omitempty"` Resources Resources `json:"resources,omitempty"` @@ -229,11 +232,12 @@ func main() { for shortName, imageConfig := range externalImageConfig.Images { var images []string isRegex, name := false, shortName - if imageConfig.ImageRegex != "" && imageConfig.Image == "" { + if (imageConfig.ImageRegex != "" || imageConfig.ImageFamily != "") && imageConfig.Image == "" { isRegex = true - images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.Project, imageConfig.PreviousImages) + images, err = getGCEImages(imageConfig.ImageRegex, imageConfig.ImageFamily, imageConfig.Project, imageConfig.PreviousImages) if err != nil { - klog.Fatalf("Could not retrieve list of images based on image prefix %q: %v", imageConfig.ImageRegex, err) + klog.Fatalf("Could not retrieve list of images based on image prefix %q and family %q: %v", + imageConfig.ImageRegex, imageConfig.ImageFamily, err) } } else { images = []string{imageConfig.Image} @@ -468,26 +472,33 @@ func (a byCreationTime) Less(i, j int) bool { return a[i].creationTime.After(a[j func (a byCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } // Returns a list of image names based on regex and number of previous images requested. -func getGCEImages(imageRegex, project string, previousImages int) ([]string, error) { - ilc, err := computeService.Images.List(project).Do() - if err != nil { - return nil, fmt.Errorf("Failed to list images in project %q: %v", project, err) - } +func getGCEImages(imageRegex, imageFamily string, project string, previousImages int) ([]string, error) { imageObjs := []imageObj{} imageRe := regexp.MustCompile(imageRegex) - for _, instance := range ilc.Items { - if imageRe.MatchString(instance.Name) { - creationTime, err := time.Parse(time.RFC3339, instance.CreationTimestamp) - if err != nil { - return nil, fmt.Errorf("Failed to parse instance creation timestamp %q: %v", instance.CreationTimestamp, err) + if err := computeService.Images.List(project).Pages(context.Background(), + func(ilc *compute.ImageList) error { + for _, instance := range ilc.Items { + if imageRegex != "" && !imageRe.MatchString(instance.Name) { + continue + } + if imageFamily != "" && instance.Family != imageFamily { + continue + } + creationTime, err := time.Parse(time.RFC3339, instance.CreationTimestamp) + if err != nil { + return fmt.Errorf("failed to parse instance creation timestamp %q: %v", instance.CreationTimestamp, err) + } + io := imageObj{ + creationTime: creationTime, + name: instance.Name, + } + klog.V(4).Infof("Found image %q based on regex %q and family %q in project %q", io.string(), imageRegex, imageFamily, project) + imageObjs = append(imageObjs, io) } - io := imageObj{ - creationTime: creationTime, - name: instance.Name, - } - klog.V(4).Infof("Found image %q based on regex %q in project %q", io.string(), imageRegex, project) - imageObjs = append(imageObjs, io) - } + return nil + }, + ); err != nil { + return nil, fmt.Errorf("failed to list images in project %q: %v", project, err) } sort.Sort(byCreationTime(imageObjs)) images := []string{} @@ -547,7 +558,13 @@ func testImage(imageConfig *internalGCEImage, junitFilePrefix string) *TestResul // Provision a gce instance using image func createInstance(imageConfig *internalGCEImage) (string, error) { - klog.V(1).Infof("Creating instance %+v", *imageConfig) + p, err := computeService.Projects.Get(*project).Do() + if err != nil { + return "", fmt.Errorf("failed to get project info %q", *project) + } + // Use default service account + serviceAccount := p.DefaultServiceAccount + klog.V(1).Infof("Creating instance %+v with service account %q", *imageConfig, serviceAccount) name := imageToInstanceName(imageConfig) i := &compute.Instance{ Name: name, @@ -572,6 +589,14 @@ func createInstance(imageConfig *internalGCEImage) (string, error) { }, }, }, + ServiceAccounts: []*compute.ServiceAccount{ + { + Email: serviceAccount, + Scopes: []string{ + "https://www.googleapis.com/auth/cloud-platform", + }, + }, + }, } for _, accelerator := range imageConfig.resources.Accelerators { @@ -591,7 +616,6 @@ func createInstance(imageConfig *internalGCEImage) (string, error) { i.GuestAccelerators = append(i.GuestAccelerators, ac) } - var err error i.Metadata = imageConfig.metadata if _, err := computeService.Instances.Get(*project, *zone, i.Name).Do(); err != nil { op, err := computeService.Instances.Insert(*project, *zone, i).Do() diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index c22b7e2a2ea..5392c8b657b 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -260,7 +260,6 @@ func (e *E2EServices) startKubelet() (*server, error) { "--kubeconfig", kubeconfigPath, "--root-dir", KubeletRootDirectory, "--v", LogVerbosityLevel, "--logtostderr", - "--allow-privileged=true", ) // Apply test framework feature gates by default. This could also be overridden diff --git a/test/integration/BUILD b/test/integration/BUILD index acce949a91e..4d657752613 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -45,6 +45,7 @@ filegroup( "//test/integration/daemonset:all-srcs", "//test/integration/defaulttolerationseconds:all-srcs", "//test/integration/deployment:all-srcs", + "//test/integration/disruption:all-srcs", "//test/integration/dryrun:all-srcs", "//test/integration/etcd:all-srcs", "//test/integration/evictions:all-srcs", diff --git a/test/integration/disruption/BUILD b/test/integration/disruption/BUILD new file mode 100644 index 00000000000..8e0ad029dd2 --- /dev/null +++ b/test/integration/disruption/BUILD @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = [ + "disruption_test.go", + "main_test.go", + ], + tags = ["integration"], + deps = [ + "//cmd/kube-apiserver/app/testing:go_default_library", + "//pkg/controller/disruption:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/discovery/cached/memory:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/restmapper:go_default_library", + "//staging/src/k8s.io/client-go/scale:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//test/integration/etcd:go_default_library", + "//test/integration/framework:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/integration/disruption/disruption_test.go b/test/integration/disruption/disruption_test.go new file mode 100644 index 00000000000..1edeecceb4a --- /dev/null +++ b/test/integration/disruption/disruption_test.go @@ -0,0 +1,284 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package disruption + +import ( + "fmt" + "testing" + "time" + + "k8s.io/api/core/v1" + "k8s.io/api/policy/v1beta1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + cacheddiscovery "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" + "k8s.io/client-go/tools/cache" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/controller/disruption" + "k8s.io/kubernetes/test/integration/etcd" + "k8s.io/kubernetes/test/integration/framework" +) + +func setup(t *testing.T) (*kubeapiservertesting.TestServer, *disruption.DisruptionController, informers.SharedInformerFactory, clientset.Interface, *apiextensionsclientset.Clientset, dynamic.Interface) { + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins", "ServiceAccount"}, framework.SharedEtcd()) + + clientSet, err := clientset.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("Error creating clientset: %v", err) + } + resyncPeriod := 12 * time.Hour + informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(server.ClientConfig, "pdb-informers")), resyncPeriod) + + client := clientset.NewForConfigOrDie(restclient.AddUserAgent(server.ClientConfig, "disruption-controller")) + + discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery()) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + + scaleKindResolver := scale.NewDiscoveryScaleKindResolver(client.Discovery()) + scaleClient, err := scale.NewForConfig(server.ClientConfig, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) + if err != nil { + t.Fatalf("Error creating scaleClient: %v", err) + } + + apiExtensionClient, err := apiextensionsclientset.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("Error creating extension clientset: %v", err) + } + + dynamicClient, err := dynamic.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("Error creating dynamicClient: %v", err) + } + + pdbc := disruption.NewDisruptionController( + informers.Core().V1().Pods(), + informers.Policy().V1beta1().PodDisruptionBudgets(), + informers.Core().V1().ReplicationControllers(), + informers.Apps().V1().ReplicaSets(), + informers.Apps().V1().Deployments(), + informers.Apps().V1().StatefulSets(), + client, + mapper, + scaleClient, + ) + return server, pdbc, informers, clientSet, apiExtensionClient, dynamicClient +} + +func TestPDBWithScaleSubresource(t *testing.T) { + s, pdbc, informers, clientSet, apiExtensionClient, dynamicClient := setup(t) + defer s.TearDownFn() + + nsName := "pdb-scale-subresource" + createNs(t, nsName, clientSet) + + stopCh := make(chan struct{}) + informers.Start(stopCh) + go pdbc.Run(stopCh) + defer close(stopCh) + + crdDefinition := newCustomResourceDefinition() + etcd.CreateTestCRDs(t, apiExtensionClient, true, crdDefinition) + gvr := schema.GroupVersionResource{Group: crdDefinition.Spec.Group, Version: crdDefinition.Spec.Version, Resource: crdDefinition.Spec.Names.Plural} + resourceClient := dynamicClient.Resource(gvr).Namespace(nsName) + + replicas := 4 + maxUnavailable := int32(2) + podLabelValue := "test-crd" + + resource := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crdDefinition.Spec.Names.Kind, + "apiVersion": crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Version, + "metadata": map[string]interface{}{ + "name": "resource", + "namespace": nsName, + }, + "spec": map[string]interface{}{ + "replicas": replicas, + }, + }, + } + createdResource, err := resourceClient.Create(resource, metav1.CreateOptions{}) + if err != nil { + t.Error(err) + } + + trueValue := true + ownerRef := metav1.OwnerReference{ + Name: resource.GetName(), + Kind: crdDefinition.Spec.Names.Kind, + APIVersion: crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Version, + UID: createdResource.GetUID(), + Controller: &trueValue, + } + for i := 0; i < replicas; i++ { + createPod(t, fmt.Sprintf("pod-%d", i), nsName, podLabelValue, clientSet, ownerRef) + } + + waitToObservePods(t, informers.Core().V1().Pods().Informer(), 4, v1.PodRunning) + + pdb := &v1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pdb", + }, + Spec: v1beta1.PodDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: maxUnavailable, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": podLabelValue}, + }, + }, + } + if _, err := clientSet.PolicyV1beta1().PodDisruptionBudgets(nsName).Create(pdb); err != nil { + t.Errorf("Error creating PodDisruptionBudget: %v", err) + } + + waitPDBStable(t, clientSet, 4, nsName, pdb.Name) + + newPdb, err := clientSet.PolicyV1beta1().PodDisruptionBudgets(nsName).Get(pdb.Name, metav1.GetOptions{}) + + if expected, found := int32(replicas), newPdb.Status.ExpectedPods; expected != found { + t.Errorf("Expected %d, but found %d", expected, found) + } + if expected, found := int32(replicas)-maxUnavailable, newPdb.Status.DesiredHealthy; expected != found { + t.Errorf("Expected %d, but found %d", expected, found) + } + if expected, found := maxUnavailable, newPdb.Status.PodDisruptionsAllowed; expected != found { + t.Errorf("Expected %d, but found %d", expected, found) + } +} + +func createPod(t *testing.T, name, namespace, labelValue string, clientSet clientset.Interface, ownerRef metav1.OwnerReference) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{"app": labelValue}, + OwnerReferences: []metav1.OwnerReference{ + ownerRef, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "fake-name", + Image: "fakeimage", + }, + }, + }, + } + _, err := clientSet.CoreV1().Pods(namespace).Create(pod) + if err != nil { + t.Error(err) + } + addPodConditionReady(pod) + if _, err := clientSet.CoreV1().Pods(namespace).UpdateStatus(pod); err != nil { + t.Error(err) + } +} + +func createNs(t *testing.T, name string, clientSet clientset.Interface) { + _, err := clientSet.CoreV1().Namespaces().Create(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }) + if err != nil { + t.Errorf("Error creating namespace: %v", err) + } +} + +func addPodConditionReady(pod *v1.Pod) { + pod.Status = v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + } +} + +func newCustomResourceDefinition() *apiextensionsv1beta1.CustomResourceDefinition { + return &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "crds.mygroup.example.com"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "mygroup.example.com", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "crds", + Singular: "crd", + Kind: "Crd", + ListKind: "CrdList", + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{ + SpecReplicasPath: ".spec.replicas", + StatusReplicasPath: ".status.replicas", + }, + }, + }, + } +} + +func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) { + if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { + pdb, err := clientSet.PolicyV1beta1().PodDisruptionBudgets(ns).Get(pdbName, metav1.GetOptions{}) + if err != nil { + return false, err + } + if pdb.Status.CurrentHealthy != podNum { + return false, nil + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} + +func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int, phase v1.PodPhase) { + if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { + objects := podInformer.GetIndexer().List() + if len(objects) != podNum { + return false, nil + } + for _, obj := range objects { + pod := obj.(*v1.Pod) + if pod.Status.Phase != phase { + return false, nil + } + } + return true, nil + }); err != nil { + t.Fatal(err) + } +} diff --git a/test/integration/disruption/main_test.go b/test/integration/disruption/main_test.go new file mode 100644 index 00000000000..5e481cf889c --- /dev/null +++ b/test/integration/disruption/main_test.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package disruption + +import ( + "k8s.io/kubernetes/test/integration/framework" + "testing" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +} diff --git a/test/integration/evictions/BUILD b/test/integration/evictions/BUILD index 153b79c25ba..edd4961be36 100644 --- a/test/integration/evictions/BUILD +++ b/test/integration/evictions/BUILD @@ -22,9 +22,13 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/discovery/cached/memory:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/restmapper:go_default_library", + "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//test/integration/framework:go_default_library", ], diff --git a/test/integration/evictions/evictions_test.go b/test/integration/evictions/evictions_test.go index 479a8075d13..a0be6e50287 100644 --- a/test/integration/evictions/evictions_test.go +++ b/test/integration/evictions/evictions_test.go @@ -18,6 +18,7 @@ package evictions import ( "fmt" + "net/http/httptest" "reflect" "sync" @@ -32,9 +33,13 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + cacheddiscovery "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/controller/disruption" "k8s.io/kubernetes/test/integration/framework" @@ -329,6 +334,17 @@ func rmSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *disruption.D resyncPeriod := 12 * time.Hour informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), resyncPeriod) + client := clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")) + + discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery()) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + + scaleKindResolver := scale.NewDiscoveryScaleKindResolver(client.Discovery()) + scaleClient, err := scale.NewForConfig(&config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) + if err != nil { + t.Fatalf("Error in create scaleClient: %v", err) + } + rm := disruption.NewDisruptionController( informers.Core().V1().Pods(), informers.Policy().V1beta1().PodDisruptionBudgets(), @@ -336,7 +352,9 @@ func rmSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *disruption.D informers.Apps().V1().ReplicaSets(), informers.Apps().V1().Deployments(), informers.Apps().V1().StatefulSets(), - clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")), + client, + mapper, + scaleClient, ) return s, closeFn, rm, informers, clientSet } diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 2ed3784d125..75db01ac1f7 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -135,7 +135,7 @@ func TestCRD(t *testing.T) { } func TestCRDOpenAPI(t *testing.T) { - result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=CustomResourcePublishOpenAPI=true"}, framework.SharedEtcd()) + result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer result.TearDownFn() kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) if err != nil { @@ -160,6 +160,7 @@ func TestCRDOpenAPI(t *testing.T) { }, Validation: &apiextensionsv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ "foo": {Type: "string"}, }, diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 439f227fddb..bdcef5c5779 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -109,12 +109,16 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/client-go/discovery/cached/memory:go_default_library", + "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/restmapper:go_default_library", + "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//test/integration/framework:go_default_library", "//test/utils/image:go_default_library", diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go index 6baf474c340..6e019ee3811 100644 --- a/test/integration/scheduler/preemption_test.go +++ b/test/integration/scheduler/preemption_test.go @@ -736,7 +736,7 @@ func TestPDBInPreemption(t *testing.T) { defer cleanupTest(t, context) cs := context.clientSet - initDisruptionController(context) + initDisruptionController(t, context) defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index 9ba8d0b8f1e..478757509f2 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -33,12 +33,16 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" + cacheddiscovery "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" clientv1core "k8s.io/client-go/kubernetes/typed/core/v1" corelisters "k8s.io/client-go/listers/core/v1" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/scale" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" @@ -238,9 +242,19 @@ func initTestSchedulerWithOptions( // initDisruptionController initializes and runs a Disruption Controller to properly // update PodDisuptionBudget objects. -func initDisruptionController(context *testContext) *disruption.DisruptionController { +func initDisruptionController(t *testing.T, context *testContext) *disruption.DisruptionController { informers := informers.NewSharedInformerFactory(context.clientSet, 12*time.Hour) + discoveryClient := cacheddiscovery.NewMemCacheClient(context.clientSet.Discovery()) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) + + config := restclient.Config{Host: context.httpServer.URL} + scaleKindResolver := scale.NewDiscoveryScaleKindResolver(context.clientSet.Discovery()) + scaleClient, err := scale.NewForConfig(&config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) + if err != nil { + t.Fatalf("Error in create scaleClient: %v", err) + } + dc := disruption.NewDisruptionController( informers.Core().V1().Pods(), informers.Policy().V1beta1().PodDisruptionBudgets(), @@ -248,7 +262,9 @@ func initDisruptionController(context *testContext) *disruption.DisruptionContro informers.Apps().V1().ReplicaSets(), informers.Apps().V1().Deployments(), informers.Apps().V1().StatefulSets(), - context.clientSet) + context.clientSet, + mapper, + scaleClient) informers.Start(context.schedulerConfig.StopEverything) informers.WaitForCacheSync(context.schedulerConfig.StopEverything) diff --git a/test/integration/statefulset/util.go b/test/integration/statefulset/util.go index bf866f7db90..3481351a1f8 100644 --- a/test/integration/statefulset/util.go +++ b/test/integration/statefulset/util.go @@ -43,9 +43,6 @@ import ( const ( pollInterval = 100 * time.Millisecond pollTimeout = 60 * time.Second - - fakeImageName = "fake-name" - fakeImage = "fakeimage" ) func labelMap() map[string]string { diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index 2b47de6a8bd..49800ec4830 100755 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -257,7 +257,6 @@ function load-docker-images { # Computes command line arguments to be passed to kubelet. function compute-kubelet-params { local params="${KUBELET_TEST_ARGS:-}" - params+=" --allow-privileged=true" params+=" --cgroup-root=/" params+=" --cloud-provider=gce" params+=" --pod-manifest-path=/etc/kubernetes/manifests" diff --git a/vendor/k8s.io/klog/klog.go b/vendor/k8s.io/klog/klog.go index 887ea62dff7..a3dddeadfcf 100644 --- a/vendor/k8s.io/klog/klog.go +++ b/vendor/k8s.io/klog/klog.go @@ -404,21 +404,36 @@ func init() { go logging.flushDaemon() } -// InitFlags is for explicitly initializing the flags +var initDefaultsOnce sync.Once + +// InitFlags is for explicitly initializing the flags. func InitFlags(flagset *flag.FlagSet) { + + // Initialize defaults. + initDefaultsOnce.Do(func() { + logging.logDir = "" + logging.logFile = "" + logging.logFileMaxSizeMB = 1800 + logging.toStderr = true + logging.alsoToStderr = false + logging.skipHeaders = false + logging.skipLogHeaders = false + }) + if flagset == nil { flagset = flag.CommandLine } - flagset.StringVar(&logging.logDir, "log_dir", "", "If non-empty, write log files in this directory") - flagset.StringVar(&logging.logFile, "log_file", "", "If non-empty, use this log file") - flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", 1800, + + flagset.StringVar(&logging.logDir, "log_dir", logging.logDir, "If non-empty, write log files in this directory") + flagset.StringVar(&logging.logFile, "log_file", logging.logFile, "If non-empty, use this log file") + flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", logging.logFileMaxSizeMB, "Defines the maximum size a log file can grow to. Unit is megabytes. "+ "If the value is 0, the maximum file size is unlimited.") - flagset.BoolVar(&logging.toStderr, "logtostderr", true, "log to standard error instead of files") - flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") + flagset.BoolVar(&logging.toStderr, "logtostderr", logging.toStderr, "log to standard error instead of files") + flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", logging.alsoToStderr, "log to standard error as well as files") flagset.Var(&logging.verbosity, "v", "number for the log level verbosity") - flagset.BoolVar(&logging.skipHeaders, "skip_headers", false, "If true, avoid header prefixes in the log messages") - flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", false, "If true, avoid headers when openning log files") + flagset.BoolVar(&logging.skipHeaders, "skip_headers", logging.skipHeaders, "If true, avoid header prefixes in the log messages") + flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", logging.skipLogHeaders, "If true, avoid headers when opening log files") flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") diff --git a/vendor/modules.txt b/vendor/modules.txt index 06313d91c02..8de917a61e5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1581,7 +1581,7 @@ k8s.io/gengo/parser k8s.io/gengo/types # k8s.io/heapster v1.2.0-beta.1 => k8s.io/heapster v1.2.0-beta.1 k8s.io/heapster/metrics/api/v1/types -# k8s.io/klog v0.3.0 => k8s.io/klog v0.3.0 +# k8s.io/klog v0.3.1 => k8s.io/klog v0.3.1 k8s.io/klog # k8s.io/kube-aggregator v0.0.0 => ./staging/src/k8s.io/kube-aggregator k8s.io/kube-aggregator/pkg/apis/apiregistration